[MNT] merge branch main into feature/hetero

.github/workflows/install_learnware_with_pip.yaml

@@ -39,11 +39,6 @@ jobs:
         conda run -n learnware python -m pip install --upgrade pip
         conda run -n learnware python -m pip install pytest
 
-    - name: Install faiss for MacOS
-      if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }}
-      run: |
-        conda run -n learnware conda install -c pytorch faiss
-
     - name: Install learnware
       run: |
         conda run -n learnware python -m pip install learnware

.github/workflows/install_learnware_with_source.yaml

@@ -44,11 +44,6 @@ jobs:
         # stop the build if there are Python syntax errors or undefined names
         conda run -n learnware python -m flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
 
-    - name: Install faiss for MacOS
-      if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }}
-      run: |
-        conda run -n learnware conda install -c pytorch faiss
-
     - name: Install learnware
       run: |
         conda run -n learnware python -m pip install .

examples/dataset_m5_workflow/example_init.py

@@ -7,7 +7,7 @@ from learnware.model import BaseModel
 
 class Model(BaseModel):
     def __init__(self):
-        super(Model, self).__init__(input_shape=(82,), output_shape=())
+        super(Model, self).__init__(input_shape=(82,), output_shape=(1,))
         dir_path = os.path.dirname(os.path.abspath(__file__))
         self.model = lgb.Booster(model_file=os.path.join(dir_path, "model.out"))
 

examples/dataset_m5_workflow/main.py

@@ -8,7 +8,6 @@ from shutil import copyfile, rmtree
 
 import learnware
 from learnware.market import instantiate_learnware_market, BaseUserInfo
-from learnware.market import database_ops
 from learnware.reuse import JobSelectorReuser, AveragingReuser
 from learnware.specification import generate_rkme_spec
 from m5 import DataLoader
@@ -17,24 +16,38 @@ from learnware.logger import get_module_logger
 logger = get_module_logger("m5_test", level="INFO")
 
 
+output_description = {
+    "Dimension": 1,
+    "Description": {},
+}
+
+input_description = {
+    "Dimension": 82,
+    "Description": {},
+}
+
 semantic_specs = [
     {
-        "Data": {"Values": ["Tabular"], "Type": "Class"},
-        "Task": {"Values": ["Classification"], "Type": "Class"},
+        "Data": {"Values": ["Table"], "Type": "Class"},
+        "Task": {"Values": ["Regression"], "Type": "Class"},
         "Library": {"Values": ["Scikit-learn"], "Type": "Class"},
         "Scenario": {"Values": ["Business"], "Type": "Tag"},
         "Description": {"Values": "", "Type": "String"},
         "Name": {"Values": "learnware_1", "Type": "String"},
+        "Input": input_description,
+        "Output": output_description,
     }
 ]
 
 user_semantic = {
-    "Data": {"Values": ["Tabular"], "Type": "Class"},
-    "Task": {"Values": ["Classification"], "Type": "Class"},
+    "Data": {"Values": ["Table"], "Type": "Class"},
+    "Task": {"Values": ["Regression"], "Type": "Class"},
     "Library": {"Values": ["Scikit-learn"], "Type": "Class"},
     "Scenario": {"Values": ["Business"], "Type": "Tag"},
     "Description": {"Values": "", "Type": "String"},
     "Name": {"Values": "", "Type": "String"},
+    "Input": input_description,
+    "Output": output_description,
 }
 
 
@@ -69,8 +82,6 @@ class M5DatasetWorkflow:
             easy_market.add_learnware(zip_path, semantic_spec)
 
         print("Total Item:", len(easy_market))
-        curr_inds = easy_market._get_ids()
-        print("Available ids:", curr_inds)
 
     def prepare_learnware(self, regenerate_flag=False):
         if regenerate_flag:
@@ -171,7 +182,7 @@ class M5DatasetWorkflow:
             job_selector_score = m5.score(test_y, job_selector_predict_y)
             print(f"mixture reuse loss (job selector): {job_selector_score}")
 
-            reuse_ensemble = AveragingReuser(learnware_list=mixture_learnware_list, mode="vote")
+            reuse_ensemble = AveragingReuser(learnware_list=mixture_learnware_list, mode="vote_by_prob")
             ensemble_predict_y = reuse_ensemble.predict(user_data=test_x)
             ensemble_score = m5.score(test_y, ensemble_predict_y)
             print(f"mixture reuse loss (ensemble): {ensemble_score}\n")

examples/dataset_pfs_workflow/example_init.py

@@ -6,7 +6,7 @@ from learnware.model import BaseModel
 
 class Model(BaseModel):
     def __init__(self):
-        super(Model, self).__init__(input_shape=(31,), output_shape=())
+        super(Model, self).__init__(input_shape=(31,), output_shape=(1,))
         dir_path = os.path.dirname(os.path.abspath(__file__))
         self.model = joblib.load(os.path.join(dir_path, "model.out"))
 

examples/dataset_pfs_workflow/main.py

@@ -15,25 +15,38 @@ from learnware.logger import get_module_logger
 
 logger = get_module_logger("pfs_test", level="INFO")
 
+output_description = {
+    "Dimension": 1,
+    "Description": {},
+}
+
+input_description = {
+    "Dimension": 31,
+    "Description": {},
+}
 
 semantic_specs = [
     {
-        "Data": {"Values": ["Tabular"], "Type": "Class"},
-        "Task": {"Values": ["Classification"], "Type": "Class"},
+        "Data": {"Values": ["Table"], "Type": "Class"},
+        "Task": {"Values": ["Regression"], "Type": "Class"},
         "Library": {"Values": ["Scikit-learn"], "Type": "Class"},
         "Scenario": {"Values": ["Business"], "Type": "Tag"},
         "Description": {"Values": "", "Type": "String"},
         "Name": {"Values": "learnware_1", "Type": "String"},
+        "Input": input_description,
+        "Output": output_description,
     }
 ]
 
 user_semantic = {
-    "Data": {"Values": ["Tabular"], "Type": "Class"},
-    "Task": {"Values": ["Classification"], "Type": "Class"},
+    "Data": {"Values": ["Table"], "Type": "Class"},
+    "Task": {"Values": ["Regression"], "Type": "Class"},
     "Library": {"Values": ["Scikit-learn"], "Type": "Class"},
     "Scenario": {"Values": ["Business"], "Type": "Tag"},
     "Description": {"Values": "", "Type": "String"},
     "Name": {"Values": "", "Type": "String"},
+    "Input": input_description,
+    "Output": output_description,
 }
 
 
@@ -42,7 +55,7 @@ class PFSDatasetWorkflow:
         pfs = Dataloader()
         pfs.regenerate_data()
 
-        algo_list = ["ridge", "lgb"]
+        algo_list = ["ridge"]  # "ridge", "lgb"
         for algo in algo_list:
             pfs.set_algo(algo)
             pfs.retrain_models()
@@ -66,8 +79,6 @@ class PFSDatasetWorkflow:
             easy_market.add_learnware(zip_path, semantic_spec)
 
         print("Total Item:", len(easy_market))
-        curr_inds = easy_market._get_ids()
-        print("Available ids:", curr_inds)
 
     def prepare_learnware(self, regenerate_flag=False):
         if regenerate_flag:
@@ -75,7 +86,7 @@ class PFSDatasetWorkflow:
 
         pfs = Dataloader()
         idx_list = pfs.get_idx_list()
-        algo_list = ["lgb"]  # ["ridge", "lgb"]
+        algo_list = ["ridge"]  # ["ridge", "lgb"]
 
         curr_root = os.path.dirname(os.path.abspath(__file__))
         curr_root = os.path.join(curr_root, "learnware_pool")
@@ -157,7 +168,7 @@ class PFSDatasetWorkflow:
                 pred_y = learnware.predict(test_x)
                 loss_list.append(pfs.score(test_y, pred_y))
             print(
-                f"Top1-score: {sorted_score_list[0]}, learnware_id: {single_learnware_list[0].id}, loss: {loss_list[0]}"
+                f"Top1-score: {sorted_score_list[0]}, learnware_id: {single_learnware_list[0].id}, loss: {loss_list[0]}, random: {np.mean(loss_list)}"
             )
 
             mixture_id = " ".join([learnware.id for learnware in mixture_learnware_list])

examples/dataset_text_workflow/main.py

@@ -196,10 +196,6 @@ def test_search(gamma=0.1, load_market=True):
         ensemble_score_list.append(ensemble_score)
         print(f"mixture reuse accuracy (ensemble): {ensemble_score}")
 
-        select_list.append(acc_list[0])
-        avg_list.append(np.mean(acc_list))
-        improve_list.append((acc_list[0] - np.mean(acc_list)) / np.mean(acc_list))
-
         # test reuse (ensemblePruning)
         reuse_pruning = EnsemblePruningReuser(learnware_list=mixture_learnware_list)
         pruning_predict_y = reuse_pruning.predict(user_data=user_data)

examples/dataset_text_workflow2/get_data.py

@@ -8,7 +8,9 @@ def get_data(data_root="./data"):
     dtest = pd.read_csv(os.path.join(data_root, "test.csv"))
 
     # returned X(DataFrame), y(Series)
-    return (dtrain[['discourse_text', 'discourse_type']],
-            dtrain["discourse_effectiveness"],
-            dtest[['discourse_text', 'discourse_type']],
-            dtest["discourse_effectiveness"])
+    return (
+        dtrain[["discourse_text", "discourse_type"]],
+        dtrain["discourse_effectiveness"],
+        dtest[["discourse_text", "discourse_type"]],
+        dtest["discourse_effectiveness"],
+    )

examples/dataset_text_workflow2/main.py

@@ -78,10 +78,10 @@ def prepare_model():
         modelv_save_path = os.path.join(model_save_root, "uploader_v_%d.pth" % (i))
         modell_save_path = os.path.join(model_save_root, "uploader_l_%d.pth" % (i))
 
-        with open(modelv_save_path, 'wb') as f:
+        with open(modelv_save_path, "wb") as f:
             pickle.dump(vectorizer, f)
 
-        with open(modell_save_path, 'wb') as f:
+        with open(modell_save_path, "wb") as f:
             pickle.dump(lgbm, f)
 
         logger.info("Model saved to '%s' and '%s'" % (modelv_save_path, modell_save_path))

examples/dataset_text_workflow2/utils.py

@@ -39,11 +39,11 @@ def generate_uploader(data_x: pd.Series, data_y: pd.Series, n_uploaders=50, data
         return
     os.makedirs(data_save_root, exist_ok=True)
 
-    types = data_x['discourse_type'].unique()
+    types = data_x["discourse_type"].unique()
 
     for i in range(n_uploaders):
-        indices = data_x['discourse_type'] == types[i]
-        selected_X = data_x[indices]['discourse_text'].to_list()
+        indices = data_x["discourse_type"] == types[i]
+        selected_X = data_x[indices]["discourse_text"].to_list()
         selected_y = data_y[indices].to_list()
 
         X_save_dir = os.path.join(data_save_root, "uploader_%d_X.pkl" % (i))
@@ -61,11 +61,11 @@ def generate_user(data_x, data_y, n_users=50, data_save_root=None):
         return
     os.makedirs(data_save_root, exist_ok=True)
 
-    types = data_x['discourse_type'].unique()
+    types = data_x["discourse_type"].unique()
 
     for i in range(n_users):
-        indices = data_x['discourse_type'] == types[i]
-        selected_X = data_x[indices]['discourse_text'].to_list()
+        indices = data_x["discourse_type"] == types[i]
+        selected_X = data_x[indices]["discourse_text"].to_list()
         selected_y = data_y[indices].to_list()
 
         X_save_dir = os.path.join(data_save_root, "user_%d_X.pkl" % (i))
@@ -80,10 +80,10 @@ def generate_user(data_x, data_y, n_users=50, data_save_root=None):
 
 # Train Uploaders' models
 def train(X, y, out_classes):
-    vectorizer = TfidfVectorizer(stop_words='english')
+    vectorizer = TfidfVectorizer(stop_words="english")
     X_tfidf = vectorizer.fit_transform(X)
 
-    lgbm = LGBMClassifier(boosting_type='dart', n_estimators=500, num_leaves=21)
+    lgbm = LGBMClassifier(boosting_type="dart", n_estimators=500, num_leaves=21)
     lgbm.fit(X_tfidf, y)
 
     return vectorizer, lgbm

learnware/client/container.py

@@ -340,7 +340,6 @@ class ModelDockerContainer(ModelContainer):
                                 "install",
                                 "-r",
                                 f"{requirements_path_filter}",
-                                "--no-dependencies",
                             ]
                         )
                     )

learnware/client/utils.py

@@ -81,7 +81,6 @@ def install_environment(learnware_dirpath, conda_env):
                     "install",
                     "-r",
                     f"{requirements_path_filter}",
-                    "--no-dependencies",
                 ]
             )
         else:

learnware/market/easy/checker.py

@@ -89,7 +89,6 @@ class EasyStatChecker(BaseChecker):
             traceback.print_exc()
             logger.warning(f"The learnware [{learnware.id}] is instantiated failed! Due to {e}.")
             return self.INVALID_LEARNWARE
-
         try:
             learnware_model = learnware.get_model()
             # Check input shape

learnware/market/easy/searcher.py

@@ -1,14 +1,13 @@
 import torch
 import numpy as np
 from rapidfuzz import fuzz
-from cvxopt import solvers, matrix
 from typing import Tuple, List, Union
 
 from .organizer import EasyOrganizer
 from ..utils import parse_specification_type
 from ..base import BaseUserInfo, BaseSearcher
 from ...learnware import Learnware
-from ...specification import RKMETableSpecification, RKMEImageSpecification, RKMETextSpecification
+from ...specification import RKMETableSpecification, RKMEImageSpecification, RKMETextSpecification, rkme_solve_qp
 from ...logger import get_module_logger
 
 logger = get_module_logger("easy_seacher")
@@ -277,19 +276,9 @@ class EasyStatSearcher(BaseSearcher):
         # weight = torch.linalg.inv(K + torch.eye(K.shape[0]).to(user_rkme.device) * 1e-5) @ C
 
         # beta must be nonnegative
-        n = K.shape[0]
-        P = matrix(K.cpu().numpy())
-        q = matrix(-C.cpu().numpy())
-        G = matrix(-np.eye(n))
-        h = matrix(np.zeros((n, 1)))
-        A = matrix(np.ones((1, n)))
-        b = matrix(np.ones((1, 1)))
-        solvers.options["show_progress"] = False
-        sol = solvers.qp(P, q, G, h, A, b)
-        weight = np.array(sol["x"])
-        weight = torch.from_numpy(weight).reshape(-1).double().to(user_rkme.device)
-        score = user_rkme.inner_prod(user_rkme) + 2 * sol["primal objective"]
-
+        weight, obj = rkme_solve_qp(K, C)
+        weight = weight.double().to(user_rkme.device)
+        score = user_rkme.inner_prod(user_rkme) + 2 * obj
         return weight.detach().cpu().numpy().reshape(-1), score
 
     def _calculate_intermediate_K_and_C(
@@ -321,7 +310,7 @@ class EasyStatSearcher(BaseSearcher):
         num = intermediate_K.shape[0] - 1
         RKME_list = [learnware.specification.get_stat_spec_by_name(self.stat_spec_type) for learnware in learnware_list]
         for i in range(intermediate_K.shape[0]):
-            intermediate_K[num, i] = RKME_list[-1].inner_prod(RKME_list[i])
+            intermediate_K[num, i] = intermediate_K[i, num] = RKME_list[-1].inner_prod(RKME_list[i])
         intermediate_C[num, 0] = user_rkme.inner_prod(RKME_list[-1])
         return intermediate_K, intermediate_C
 

learnware/reuse/job_selector.py

@@ -2,7 +2,7 @@ import torch
 import numpy as np
 
 from typing import List, Union
-from cvxopt import matrix, solvers
+from qpsolvers import solve_qp
 from lightgbm import LGBMClassifier, early_stopping
 from sklearn.metrics import accuracy_score
 
@@ -11,7 +11,7 @@ from .base import BaseReuser
 from ..market.utils import parse_specification_type
 from ..learnware import Learnware
 from ..specification import RKMETableSpecification, RKMETextSpecification
-from ..specification import generate_rkme_spec
+from ..specification import generate_rkme_spec, rkme_solve_qp
 from ..logger import get_module_logger
 
 logger = get_module_logger("job_selector_reuse")
@@ -184,16 +184,8 @@ class JobSelectorReuser(BaseReuser):
         K = task_rkme_matrix
         v = np.array([user_rkme_spec.inner_prod(task_rkme) for task_rkme in task_rkme_list])
 
-        P = matrix(K)
-        q = matrix(-v)
-        G = matrix(-np.eye(task_num))
-        h = matrix(np.zeros((task_num, 1)))
-        A = matrix(np.ones((1, task_num)))
-        b = matrix(np.ones((1, 1)))
-        solvers.options["show_progress"] = False
-
-        sol = solvers.qp(P, q, G, h, A, b, kktsolver="ldl")
-        task_mixture_weight = np.array(sol["x"]).reshape(-1)
+        sol, _ = rkme_solve_qp(K, v)
+        task_mixture_weight = np.array(sol).reshape(-1)
 
         return task_mixture_weight
 

learnware/specification/__init__.py

@@ -5,6 +5,7 @@ from .regular import (
     RKMETableSpecification,
     RKMEImageSpecification,
     RKMETextSpecification,
+    rkme_solve_qp,
 )
 
 from .system import HeteroMapTableSpecification

learnware/specification/regular/__init__.py

@@ -2,5 +2,5 @@ from .base import RegularStatSpecification
 from ...utils import is_torch_avaliable
 
 from .text import RKMETextSpecification
-from .table import RKMETableSpecification, RKMEStatSpecification
+from .table import RKMETableSpecification, RKMEStatSpecification, rkme_solve_qp
 from .image import RKMEImageSpecification

learnware/specification/regular/table/__init__.py

@@ -8,4 +8,4 @@ if not is_torch_avaliable(verbose=False):
     RKMEStatSpecification = None
     logger.warning("RKMETableSpecification is skipped because torch is not installed!")
 else:
-    from .rkme import RKMETableSpecification, RKMEStatSpecification
+    from .rkme import RKMETableSpecification, RKMEStatSpecification, rkme_solve_qp

learnware/specification/regular/table/rkme.py

@@ -1,35 +1,23 @@
 from __future__ import annotations
 
 import os
-
 import copy
 import torch
 import json
 import codecs
 import random
 import numpy as np
-from cvxopt import solvers, matrix
+from qpsolvers import solve_qp, Problem, solve_problem
 from collections import Counter
 from typing import Tuple, Any, List, Union, Dict
-
-try:
-    import faiss
-
-    ver = faiss.__version__
-    _FAISS_INSTALLED = ver >= "1.7.1"
-except ImportError:
-    _FAISS_INSTALLED = False
+import scipy
+from sklearn.cluster import MiniBatchKMeans
 
 from ..base import RegularStatSpecification
 from ....logger import get_module_logger
 
 logger = get_module_logger("rkme")
 
-if not _FAISS_INSTALLED:
-    logger.warning(
-        "Required faiss version >= 1.7.1 is not detected! Please run 'conda install -c pytorch faiss-cpu' first"
-    )
-
 
 class RKMETableSpecification(RegularStatSpecification):
     """Reduced Kernel Mean Embedding (RKME) Specification"""
@@ -125,8 +113,8 @@ class RKMETableSpecification(RegularStatSpecification):
             self.beta = torch.from_numpy(self.beta).double().to(self.device)
             return
 
-        # Initialize Z by clustering, utiliing faiss to speed up the process.
-        self._init_z_by_faiss(X, K)
+        # Initialize Z by clustering, utiliing kmeans to speed up the process.
+        self._init_z_by_kmeans(X, K)
         self._update_beta(X, nonnegative_beta)
 
         # Alternating optimize Z and beta
@@ -137,8 +125,8 @@ class RKMETableSpecification(RegularStatSpecification):
         # Reshape to original dimensions
         self.z = self.z.reshape(Z_shape)
 
-    def _init_z_by_faiss(self, X: Union[np.ndarray, torch.tensor], K: int):
-        """Intialize Z by faiss clustering.
+    def _init_z_by_kmeans(self, X: Union[np.ndarray, torch.tensor], K: int):
+        """Intialize Z by kmeans clustering.
 
         Parameters
         ----------
@@ -148,10 +136,9 @@ class RKMETableSpecification(RegularStatSpecification):
             Size of the construced reduced set.
         """
         X = X.astype("float32")
-        numDim = X.shape[1]
-        kmeans = faiss.Kmeans(numDim, K, niter=100, verbose=False)
-        kmeans.train(X)
-        center = torch.from_numpy(kmeans.centroids).double()
+        kmeans = MiniBatchKMeans(n_clusters=K, max_iter=100, verbose=False, n_init="auto")
+        kmeans.fit(X)
+        center = torch.from_numpy(kmeans.cluster_centers_).double()
         self.z = center
 
     def _update_beta(self, X: Any, nonnegative_beta: bool = True):
@@ -177,7 +164,8 @@ class RKMETableSpecification(RegularStatSpecification):
         C = torch.sum(C, dim=1) / X.shape[0]
 
         if nonnegative_beta:
-            beta = solve_qp(K, C).to(self.device)
+            beta, _ = rkme_solve_qp(K, C)
+            beta = beta.to(self.device)
         else:
             beta = torch.linalg.inv(K + torch.eye(K.shape[0]).to(self.device) * 1e-5) @ C
 
@@ -536,7 +524,7 @@ def torch_rbf_kernel(x1, x2, gamma) -> torch.Tensor:
     return torch.exp(-X12norm * gamma)
 
 
-def solve_qp(K: np.ndarray, C: np.ndarray):
+def rkme_solve_qp(K: np.ndarray, C: np.ndarray):
     """Solver for the following quadratic programming(QP) problem:
         - min   1/2 x^T K x - C^T x
         s.t     1^T x - 1 = 0
@@ -554,18 +542,24 @@ def solve_qp(K: np.ndarray, C: np.ndarray):
     torch.tensor
             Solution to the QP problem.
     """
+    if torch.is_tensor(K):
+        K = K.cpu().numpy()
+    if torch.is_tensor(C):
+        C = C.cpu().numpy()
+    C = C.reshape(-1)
+
     n = K.shape[0]
-    P = matrix(K.cpu().numpy())
-    q = matrix(-C.cpu().numpy())
-    G = matrix(-np.eye(n))
-    h = matrix(np.zeros((n, 1)))
-    A = matrix(np.ones((1, n)))
-    b = matrix(np.ones((1, 1)))
-
-    solvers.options["show_progress"] = False
-    sol = solvers.qp(P, q, G, h, A, b)  # Requires the sum of x to be 1
-    # sol = solvers.qp(P, q, G, h) # Otherwise
-    w = np.array(sol["x"])
+    P = np.array(K)
+    P = scipy.sparse.csc_matrix(P)
+    q = np.array(-C)
+    G = np.array(-np.eye(n))
+    G = scipy.sparse.csc_matrix(G)
+    h = np.array(np.zeros((n, 1)))
+    A = np.array(np.ones((1, n)))
+    A = scipy.sparse.csc_matrix(A)
+    b = np.array(np.ones((1, 1)))
+    problem = Problem(P, q, G, h, A, b)
+    solution = solve_problem(problem, solver="clarabel")
+    w = solution.x
     w = torch.from_numpy(w).reshape(-1)
-
-    return w
+    return w, solution.obj

setup.py

@@ -54,7 +54,6 @@ REQUIRED = [
     "numpy>=1.20.0",
     "pandas>=0.25.1",
     "scipy>=1.0.0",
-    "cvxopt>=1.3.0",
     "tqdm>=4.65.0",
     "scikit-learn>=0.22",
     "joblib>=1.2.0",
@@ -68,11 +67,9 @@ REQUIRED = [
     "langdetect>=1.0.9",
     "huggingface-hub<0.18",
     "portalocker>=2.0.0",
+    "qpsolvers[clarabel]>=4.0.1",
 ]
 
-if get_platform() != MACOS:
-    REQUIRED.append("faiss-cpu>=1.7.1")
-
 here = os.path.abspath(os.path.dirname(__file__))
 with open(os.path.join(here, "README.md"), encoding="utf-8") as f:
     long_description = f.read()