From 1d846e5de5939e020594d5d6db00fae153c242c1 Mon Sep 17 00:00:00 2001 From: nju-xy <1582857295@qq.com> Date: Thu, 9 Nov 2023 14:46:31 +0800 Subject: [PATCH 01/10] [MNT] remove cvxopt --- learnware/market/easy.py | 20 +++-------- learnware/market/easy2/searcher.py | 16 ++------- learnware/reuse/job_selector.py | 16 +++------ learnware/specification/__init__.py | 1 + learnware/specification/regular/__init__.py | 2 +- .../specification/regular/table/__init__.py | 2 +- learnware/specification/regular/table/rkme.py | 35 ++++++++++--------- setup.py | 2 +- 8 files changed, 35 insertions(+), 59 deletions(-) diff --git a/learnware/market/easy.py b/learnware/market/easy.py index b57e1c0..9b14d4b 100644 --- a/learnware/market/easy.py +++ b/learnware/market/easy.py @@ -7,7 +7,6 @@ import traceback import numpy as np import pandas as pd from rapidfuzz import fuzz -from cvxopt import solvers, matrix from shutil import copyfile, rmtree from typing import Tuple, Any, List, Union, Dict @@ -18,7 +17,7 @@ from .. import utils from ..config import C as conf from ..logger import get_module_logger from ..learnware import Learnware, get_learnware_from_dirpath -from ..specification import RKMETableSpecification, Specification +from ..specification import RKMETableSpecification, Specification, rkme_solve_qp logger = get_module_logger("market", "INFO") @@ -347,18 +346,9 @@ class EasyMarket(LearnwareMarket): # weight = torch.linalg.inv(K + torch.eye(K.shape[0]).to(user_rkme.device) * 1e-5) @ C # beta must be nonnegative - n = K.shape[0] - P = matrix(K.cpu().numpy()) - q = matrix(-C.cpu().numpy()) - G = matrix(-np.eye(n)) - h = matrix(np.zeros((n, 1))) - A = matrix(np.ones((1, n))) - b = matrix(np.ones((1, 1))) - solvers.options["show_progress"] = False - sol = solvers.qp(P, q, G, h, A, b) - weight = np.array(sol["x"]) - weight = torch.from_numpy(weight).reshape(-1).double().to(user_rkme.device) - score = user_rkme.inner_prod(user_rkme) + 2 * sol["primal objective"] + weight, obj = rkme_solve_qp(K, C) + weight = weight.to(user_rkme.device) + score = user_rkme.inner_prod(user_rkme) + 2 * obj return weight.detach().cpu().numpy().reshape(-1), score @@ -926,7 +916,7 @@ class EasyMarket(LearnwareMarket): logger.warning("Learnware ID '%s' NOT Found!" % (ids)) return None - def get_learnware_zip_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]: + def get_learnware_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]: """Get Zipped Learnware file by id Parameters diff --git a/learnware/market/easy2/searcher.py b/learnware/market/easy2/searcher.py index f86c06c..cc51b30 100644 --- a/learnware/market/easy2/searcher.py +++ b/learnware/market/easy2/searcher.py @@ -1,14 +1,13 @@ import torch import numpy as np from rapidfuzz import fuzz -from cvxopt import solvers, matrix from typing import Tuple, List, Union from .organizer import EasyOrganizer from ..utils import parse_specification_type from ..base import BaseUserInfo, BaseSearcher from ...learnware import Learnware -from ...specification import RKMETableSpecification, RKMEImageSpecification, RKMETextSpecification +from ...specification import RKMETableSpecification, RKMEImageSpecification, RKMETextSpecification, rkme_solve_qp from ...logger import get_module_logger logger = get_module_logger("easy_seacher") @@ -277,18 +276,9 @@ class EasyStatSearcher(BaseSearcher): # weight = torch.linalg.inv(K + torch.eye(K.shape[0]).to(user_rkme.device) * 1e-5) @ C # beta must be nonnegative - n = K.shape[0] - P = matrix(K.cpu().numpy()) - q = matrix(-C.cpu().numpy()) - G = matrix(-np.eye(n)) - h = matrix(np.zeros((n, 1))) - A = matrix(np.ones((1, n))) - b = matrix(np.ones((1, 1))) - solvers.options["show_progress"] = False - sol = solvers.qp(P, q, G, h, A, b) - weight = np.array(sol["x"]) + weight, obj = rkme_solve_qp(K, C) weight = torch.from_numpy(weight).reshape(-1).double().to(user_rkme.device) - score = user_rkme.inner_prod(user_rkme) + 2 * sol["primal objective"] + score = user_rkme.inner_prod(user_rkme) + 2 * obj return weight.detach().cpu().numpy().reshape(-1), score diff --git a/learnware/reuse/job_selector.py b/learnware/reuse/job_selector.py index 5e3a71f..eb5a29a 100644 --- a/learnware/reuse/job_selector.py +++ b/learnware/reuse/job_selector.py @@ -2,7 +2,7 @@ import torch import numpy as np from typing import List, Union -from cvxopt import matrix, solvers +from qpsolvers import solve_qp from lightgbm import LGBMClassifier, early_stopping from sklearn.metrics import accuracy_score @@ -11,7 +11,7 @@ from .base import BaseReuser from ..market.utils import parse_specification_type from ..learnware import Learnware from ..specification import RKMETableSpecification, RKMETextSpecification -from ..specification import generate_rkme_spec +from ..specification import generate_rkme_spec, rkme_solve_qp from ..logger import get_module_logger logger = get_module_logger("job_selector_reuse") @@ -184,16 +184,8 @@ class JobSelectorReuser(BaseReuser): K = task_rkme_matrix v = np.array([user_rkme_spec.inner_prod(task_rkme) for task_rkme in task_rkme_list]) - P = matrix(K) - q = matrix(-v) - G = matrix(-np.eye(task_num)) - h = matrix(np.zeros((task_num, 1))) - A = matrix(np.ones((1, task_num))) - b = matrix(np.ones((1, 1))) - solvers.options["show_progress"] = False - - sol = solvers.qp(P, q, G, h, A, b, kktsolver="ldl") - task_mixture_weight = np.array(sol["x"]).reshape(-1) + sol, _ = rkme_solve_qp(K, V) + task_mixture_weight = np.array(sol).reshape(-1) return task_mixture_weight diff --git a/learnware/specification/__init__.py b/learnware/specification/__init__.py index 4ecedc3..ca4bd9b 100644 --- a/learnware/specification/__init__.py +++ b/learnware/specification/__init__.py @@ -6,4 +6,5 @@ from .regular import ( RKMETableSpecification, RKMEImageSpecification, RKMETextSpecification, + rkme_solve_qp, ) diff --git a/learnware/specification/regular/__init__.py b/learnware/specification/regular/__init__.py index 9007e4d..d923ee6 100644 --- a/learnware/specification/regular/__init__.py +++ b/learnware/specification/regular/__init__.py @@ -1,4 +1,4 @@ from .text import RKMETextSpecification -from .table import RKMETableSpecification, RKMEStatSpecification +from .table import RKMETableSpecification, RKMEStatSpecification, rkme_solve_qp from .image import RKMEImageSpecification from .base import RegularStatsSpecification diff --git a/learnware/specification/regular/table/__init__.py b/learnware/specification/regular/table/__init__.py index 19fa956..d11202e 100644 --- a/learnware/specification/regular/table/__init__.py +++ b/learnware/specification/regular/table/__init__.py @@ -1 +1 @@ -from .rkme import RKMETableSpecification, RKMEStatSpecification +from .rkme import RKMETableSpecification, RKMEStatSpecification, rkme_solve_qp diff --git a/learnware/specification/regular/table/rkme.py b/learnware/specification/regular/table/rkme.py index 17aedc1..667adb7 100644 --- a/learnware/specification/regular/table/rkme.py +++ b/learnware/specification/regular/table/rkme.py @@ -8,9 +8,11 @@ import json import codecs import random import numpy as np -from cvxopt import solvers, matrix +# from cvxopt import solvers, matrix +from qpsolvers import solve_qp, Problem, solve_problem from collections import Counter from typing import Tuple, Any, List, Union, Dict +import scipy.sparse try: import faiss @@ -177,7 +179,8 @@ class RKMETableSpecification(RegularStatsSpecification): C = torch.sum(C, dim=1) / X.shape[0] if nonnegative_beta: - beta = solve_qp(K, C).to(self.device) + beta, _ = rkme_solve_qp(K, C) + beta = beta.to(self.device) else: beta = torch.linalg.inv(K + torch.eye(K.shape[0]).to(self.device) * 1e-5) @ C @@ -536,7 +539,7 @@ def torch_rbf_kernel(x1, x2, gamma) -> torch.Tensor: return torch.exp(-X12norm * gamma) -def solve_qp(K: np.ndarray, C: np.ndarray): +def rkme_solve_qp(K: np.ndarray, C: np.ndarray): """Solver for the following quadratic programming(QP) problem: - min 1/2 x^T K x - C^T x s.t 1^T x - 1 = 0 @@ -555,17 +558,17 @@ def solve_qp(K: np.ndarray, C: np.ndarray): Solution to the QP problem. """ n = K.shape[0] - P = matrix(K.cpu().numpy()) - q = matrix(-C.cpu().numpy()) - G = matrix(-np.eye(n)) - h = matrix(np.zeros((n, 1))) - A = matrix(np.ones((1, n))) - b = matrix(np.ones((1, 1))) - - solvers.options["show_progress"] = False - sol = solvers.qp(P, q, G, h, A, b) # Requires the sum of x to be 1 - # sol = solvers.qp(P, q, G, h) # Otherwise - w = np.array(sol["x"]) + P = np.array(K.cpu().numpy()) + q = np.array(-C.cpu().numpy()) + G = np.array(-np.eye(n)) + h = np.array(np.zeros((n, 1))) + A = np.array(np.ones((1, n))) + b = np.array(np.ones((1, 1))) + + # sol = solve_qp(P, q, G, h, A, b, solver="clarabel") # Requires the sum of x to be 1 + # sol = solver_qp(P, q, G, h, solver="clarabel") # Otherwise + problem = Problem(P, q, G, h, A, b) + sol = solve_problem(problem, solver="clarabel") + w = sol.x w = torch.from_numpy(w).reshape(-1) - - return w + return w, sol.obj diff --git a/setup.py b/setup.py index 67f7254..d8dde7a 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,6 @@ REQUIRED = [ "scipy>=1.0.0", "matplotlib>=3.1.3", "torch>=1.11.0", - "cvxopt>=1.3.0", "tqdm>=4.65.0", "scikit-learn>=0.22", "joblib>=1.2.0", @@ -76,6 +75,7 @@ REQUIRED = [ "langdetect>=1.0.9", "huggingface-hub<0.18", "portalocker>=2.0.0", + "qpsolvers[clarabel]>=4.0.1" ] if get_platform() != MACOS: From b174496a6d9ca11d43fee2b882a3fe0e8304aac1 Mon Sep 17 00:00:00 2001 From: nju-xy <1582857295@qq.com> Date: Thu, 9 Nov 2023 15:07:25 +0800 Subject: [PATCH 02/10] [MNT] fix small bugs --- learnware/reuse/job_selector.py | 2 +- learnware/specification/regular/table/rkme.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/learnware/reuse/job_selector.py b/learnware/reuse/job_selector.py index eb5a29a..d9f9fbf 100644 --- a/learnware/reuse/job_selector.py +++ b/learnware/reuse/job_selector.py @@ -184,7 +184,7 @@ class JobSelectorReuser(BaseReuser): K = task_rkme_matrix v = np.array([user_rkme_spec.inner_prod(task_rkme) for task_rkme in task_rkme_list]) - sol, _ = rkme_solve_qp(K, V) + sol, _ = rkme_solve_qp(K, v) task_mixture_weight = np.array(sol).reshape(-1) return task_mixture_weight diff --git a/learnware/specification/regular/table/rkme.py b/learnware/specification/regular/table/rkme.py index 667adb7..3fcd985 100644 --- a/learnware/specification/regular/table/rkme.py +++ b/learnware/specification/regular/table/rkme.py @@ -12,7 +12,7 @@ import numpy as np from qpsolvers import solve_qp, Problem, solve_problem from collections import Counter from typing import Tuple, Any, List, Union, Dict -import scipy.sparse +import scipy try: import faiss @@ -557,12 +557,19 @@ def rkme_solve_qp(K: np.ndarray, C: np.ndarray): torch.tensor Solution to the QP problem. """ + if torch.is_tensor(K): + K = K.cpu().numpy() + if torch.is_tensor(C): + C = C.cpu().numpy() n = K.shape[0] - P = np.array(K.cpu().numpy()) - q = np.array(-C.cpu().numpy()) + P = np.array(K) + P = scipy.sparse.csc_matrix(P) + q = np.array(-C) G = np.array(-np.eye(n)) + G = scipy.sparse.csc_matrix(G) h = np.array(np.zeros((n, 1))) A = np.array(np.ones((1, n))) + A = scipy.sparse.csc_matrix(A) b = np.array(np.ones((1, 1))) # sol = solve_qp(P, q, G, h, A, b, solver="clarabel") # Requires the sum of x to be 1 From 3a22a48ae50005a8e99d32e66e7a1f4dbd7fc119 Mon Sep 17 00:00:00 2001 From: nju-xy <1582857295@qq.com> Date: Thu, 9 Nov 2023 20:36:18 +0800 Subject: [PATCH 03/10] [FIX] fix dataset_m5_workflow --- examples/dataset_m5_workflow/example_init.py | 2 +- examples/dataset_m5_workflow/main.py | 31 +++++++++++++------ examples/dataset_pfs_workflow/main.py | 27 +++++++++++----- examples/dataset_text_workflow/main.py | 4 --- learnware/market/easy/checker.py | 1 + learnware/market/easy/searcher.py | 2 +- learnware/specification/regular/table/rkme.py | 29 +++++++++++++---- setup.py | 6 ++-- 8 files changed, 71 insertions(+), 31 deletions(-) diff --git a/examples/dataset_m5_workflow/example_init.py b/examples/dataset_m5_workflow/example_init.py index e0aabdd..eade812 100644 --- a/examples/dataset_m5_workflow/example_init.py +++ b/examples/dataset_m5_workflow/example_init.py @@ -7,7 +7,7 @@ from learnware.model import BaseModel class Model(BaseModel): def __init__(self): - super(Model, self).__init__(input_shape=(82,), output_shape=()) + super(Model, self).__init__(input_shape=(82,), output_shape=(1,)) dir_path = os.path.dirname(os.path.abspath(__file__)) self.model = lgb.Booster(model_file=os.path.join(dir_path, "model.out")) diff --git a/examples/dataset_m5_workflow/main.py b/examples/dataset_m5_workflow/main.py index 7a8d971..2e126e0 100644 --- a/examples/dataset_m5_workflow/main.py +++ b/examples/dataset_m5_workflow/main.py @@ -8,7 +8,7 @@ from shutil import copyfile, rmtree import learnware from learnware.market import instantiate_learnware_market, BaseUserInfo -from learnware.market import database_ops +# from learnware.market import database_ops from learnware.reuse import JobSelectorReuser, AveragingReuser from learnware.specification import generate_rkme_spec from m5 import DataLoader @@ -17,27 +17,40 @@ from learnware.logger import get_module_logger logger = get_module_logger("m5_test", level="INFO") +output_description = { + "Dimension": 1, + "Description": {}, +} + +input_description = { + "Dimension": 82, + "Description": {}, +} + semantic_specs = [ { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": {"Values": ["Classification"], "Type": "Class"}, + "Data": {"Values": ["Table"], "Type": "Class"}, + "Task": {"Values": ["Regression"], "Type": "Class"}, "Library": {"Values": ["Scikit-learn"], "Type": "Class"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, "Name": {"Values": "learnware_1", "Type": "String"}, + "Input": input_description, + "Output": output_description, } ] user_semantic = { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": {"Values": ["Classification"], "Type": "Class"}, + "Data": {"Values": ["Table"], "Type": "Class"}, + "Task": {"Values": ["Regression"], "Type": "Class"}, "Library": {"Values": ["Scikit-learn"], "Type": "Class"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, "Name": {"Values": "", "Type": "String"}, + "Input": input_description, + "Output": output_description, } - class M5DatasetWorkflow: def _init_m5_dataset(self): m5 = DataLoader() @@ -69,8 +82,8 @@ class M5DatasetWorkflow: easy_market.add_learnware(zip_path, semantic_spec) print("Total Item:", len(easy_market)) - curr_inds = easy_market._get_ids() - print("Available ids:", curr_inds) + # curr_inds = easy_market._get_ids() + # print("Available ids:", curr_inds) def prepare_learnware(self, regenerate_flag=False): if regenerate_flag: @@ -171,7 +184,7 @@ class M5DatasetWorkflow: job_selector_score = m5.score(test_y, job_selector_predict_y) print(f"mixture reuse loss (job selector): {job_selector_score}") - reuse_ensemble = AveragingReuser(learnware_list=mixture_learnware_list, mode="vote") + reuse_ensemble = AveragingReuser(learnware_list=mixture_learnware_list, mode="vote_by_prob") ensemble_predict_y = reuse_ensemble.predict(user_data=test_x) ensemble_score = m5.score(test_y, ensemble_predict_y) print(f"mixture reuse loss (ensemble): {ensemble_score}\n") diff --git a/examples/dataset_pfs_workflow/main.py b/examples/dataset_pfs_workflow/main.py index d66074e..9ce02f7 100644 --- a/examples/dataset_pfs_workflow/main.py +++ b/examples/dataset_pfs_workflow/main.py @@ -15,25 +15,38 @@ from learnware.logger import get_module_logger logger = get_module_logger("pfs_test", level="INFO") +output_description = { + "Dimension": 1, + "Description": {}, +} + +input_description = { + "Dimension": 31, + "Description": {}, +} semantic_specs = [ { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": {"Values": ["Classification"], "Type": "Class"}, + "Data": {"Values": ["Table"], "Type": "Class"}, + "Task": {"Values": ["Regression"], "Type": "Class"}, "Library": {"Values": ["Scikit-learn"], "Type": "Class"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, "Name": {"Values": "learnware_1", "Type": "String"}, + "Input": input_description, + "Output": output_description, } ] user_semantic = { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": {"Values": ["Classification"], "Type": "Class"}, + "Data": {"Values": ["Table"], "Type": "Class"}, + "Task": {"Values": ["Regression"], "Type": "Class"}, "Library": {"Values": ["Scikit-learn"], "Type": "Class"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "", "Type": "String"}, + "Name": {"Values": "learnware_1", "Type": "String"}, + "Input": input_description, + "Output": output_description, } @@ -66,8 +79,8 @@ class PFSDatasetWorkflow: easy_market.add_learnware(zip_path, semantic_spec) print("Total Item:", len(easy_market)) - curr_inds = easy_market._get_ids() - print("Available ids:", curr_inds) + # curr_inds = easy_market._get_ids() + # print("Available ids:", curr_inds) def prepare_learnware(self, regenerate_flag=False): if regenerate_flag: diff --git a/examples/dataset_text_workflow/main.py b/examples/dataset_text_workflow/main.py index bf20091..179bf53 100644 --- a/examples/dataset_text_workflow/main.py +++ b/examples/dataset_text_workflow/main.py @@ -196,10 +196,6 @@ def test_search(gamma=0.1, load_market=True): ensemble_score_list.append(ensemble_score) print(f"mixture reuse accuracy (ensemble): {ensemble_score}") - select_list.append(acc_list[0]) - avg_list.append(np.mean(acc_list)) - improve_list.append((acc_list[0] - np.mean(acc_list)) / np.mean(acc_list)) - # test reuse (ensemblePruning) reuse_pruning = EnsemblePruningReuser(learnware_list=mixture_learnware_list) pruning_predict_y = reuse_pruning.predict(user_data=user_data) diff --git a/learnware/market/easy/checker.py b/learnware/market/easy/checker.py index 5e455a7..a988157 100644 --- a/learnware/market/easy/checker.py +++ b/learnware/market/easy/checker.py @@ -120,6 +120,7 @@ class EasyStatChecker(BaseChecker): raise ValueError(f"not supported spec type for spec_type = {spec_type}") # Check output + outputs = learnware.predict(inputs) try: outputs = learnware.predict(inputs) except Exception: diff --git a/learnware/market/easy/searcher.py b/learnware/market/easy/searcher.py index cc51b30..5fb5671 100644 --- a/learnware/market/easy/searcher.py +++ b/learnware/market/easy/searcher.py @@ -277,7 +277,7 @@ class EasyStatSearcher(BaseSearcher): # beta must be nonnegative weight, obj = rkme_solve_qp(K, C) - weight = torch.from_numpy(weight).reshape(-1).double().to(user_rkme.device) + weight = weight.double().to(user_rkme.device) score = user_rkme.inner_prod(user_rkme) + 2 * obj return weight.detach().cpu().numpy().reshape(-1), score diff --git a/learnware/specification/regular/table/rkme.py b/learnware/specification/regular/table/rkme.py index 3fcd985..8f147b2 100644 --- a/learnware/specification/regular/table/rkme.py +++ b/learnware/specification/regular/table/rkme.py @@ -13,6 +13,7 @@ from qpsolvers import solve_qp, Problem, solve_problem from collections import Counter from typing import Tuple, Any, List, Union, Dict import scipy +from sklearn.cluster import MiniBatchKMeans try: import faiss @@ -27,10 +28,10 @@ from ....logger import get_module_logger logger = get_module_logger("rkme") -if not _FAISS_INSTALLED: - logger.warning( - "Required faiss version >= 1.7.1 is not detected! Please run 'conda install -c pytorch faiss-cpu' first" - ) +# if not _FAISS_INSTALLED: +# logger.warning( +# "Required faiss version >= 1.7.1 is not detected! Please run 'conda install -c pytorch faiss-cpu' first" +# ) class RKMETableSpecification(RegularStatsSpecification): @@ -127,8 +128,8 @@ class RKMETableSpecification(RegularStatsSpecification): self.beta = torch.from_numpy(self.beta).double().to(self.device) return - # Initialize Z by clustering, utiliing faiss to speed up the process. - self._init_z_by_faiss(X, K) + # Initialize Z by clustering, utiliing kmeans or faiss to speed up the process. + self._init_z_by_kmeans(X, K) self._update_beta(X, nonnegative_beta) # Alternating optimize Z and beta @@ -156,6 +157,22 @@ class RKMETableSpecification(RegularStatsSpecification): center = torch.from_numpy(kmeans.centroids).double() self.z = center + def _init_z_by_kmeans(self, X: Union[np.ndarray, torch.tensor], K: int): + """Intialize Z by kmeans clustering. + + Parameters + ---------- + X : np.ndarray or torch.tensor + Raw data in np.ndarray format or torch.tensor format. + K : int + Size of the construced reduced set. + """ + X = X.astype("float32") + kmeans = MiniBatchKMeans(n_clusters=K, max_iter=100, verbose=False, n_init="auto") + kmeans.fit(X) + center = torch.from_numpy(kmeans.cluster_centers_).double() + self.z = center + def _update_beta(self, X: Any, nonnegative_beta: bool = True): """Fix Z and update beta using its closed-form solution. diff --git a/setup.py b/setup.py index d8dde7a..a8d6e5d 100644 --- a/setup.py +++ b/setup.py @@ -75,11 +75,11 @@ REQUIRED = [ "langdetect>=1.0.9", "huggingface-hub<0.18", "portalocker>=2.0.0", - "qpsolvers[clarabel]>=4.0.1" + "qpsolvers[clarabel]>=4.0.1", ] -if get_platform() != MACOS: - REQUIRED.append("faiss-cpu>=1.7.1") +# if get_platform() != MACOS: +# REQUIRED.append("faiss-cpu>=1.7.1") here = os.path.abspath(os.path.dirname(__file__)) with open(os.path.join(here, "README.md"), encoding="utf-8") as f: From 356375d5252f04f73f879df68d0feccb8bd35451 Mon Sep 17 00:00:00 2001 From: nju-xy <1582857295@qq.com> Date: Thu, 9 Nov 2023 21:07:36 +0800 Subject: [PATCH 04/10] [FIX] try to fix pfs --- examples/dataset_pfs_workflow/example_init.py | 2 +- examples/dataset_pfs_workflow/main.py | 6 +++--- learnware/market/easy/checker.py | 4 +--- learnware/specification/regular/table/rkme.py | 2 +- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/examples/dataset_pfs_workflow/example_init.py b/examples/dataset_pfs_workflow/example_init.py index 88b788a..77bad5e 100644 --- a/examples/dataset_pfs_workflow/example_init.py +++ b/examples/dataset_pfs_workflow/example_init.py @@ -6,7 +6,7 @@ from learnware.model import BaseModel class Model(BaseModel): def __init__(self): - super(Model, self).__init__(input_shape=(31,), output_shape=()) + super(Model, self).__init__(input_shape=(31,), output_shape=(1,)) dir_path = os.path.dirname(os.path.abspath(__file__)) self.model = joblib.load(os.path.join(dir_path, "model.out")) diff --git a/examples/dataset_pfs_workflow/main.py b/examples/dataset_pfs_workflow/main.py index 9ce02f7..1f8c348 100644 --- a/examples/dataset_pfs_workflow/main.py +++ b/examples/dataset_pfs_workflow/main.py @@ -44,7 +44,7 @@ user_semantic = { "Library": {"Values": ["Scikit-learn"], "Type": "Class"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_1", "Type": "String"}, + "Name": {"Values": "", "Type": "String"}, "Input": input_description, "Output": output_description, } @@ -55,7 +55,7 @@ class PFSDatasetWorkflow: pfs = Dataloader() pfs.regenerate_data() - algo_list = ["ridge", "lgb"] + algo_list = ["ridge"] # , "lgb" for algo in algo_list: pfs.set_algo(algo) pfs.retrain_models() @@ -88,7 +88,7 @@ class PFSDatasetWorkflow: pfs = Dataloader() idx_list = pfs.get_idx_list() - algo_list = ["lgb"] # ["ridge", "lgb"] + algo_list = ["ridge"] # ["ridge", "lgb"] curr_root = os.path.dirname(os.path.abspath(__file__)) curr_root = os.path.join(curr_root, "learnware_pool") diff --git a/learnware/market/easy/checker.py b/learnware/market/easy/checker.py index a988157..d0d50cf 100644 --- a/learnware/market/easy/checker.py +++ b/learnware/market/easy/checker.py @@ -89,7 +89,6 @@ class EasyStatChecker(BaseChecker): traceback.print_exc() logger.warning(f"The learnware [{learnware.id}] is instantiated failed! Due to {e}.") return self.INVALID_LEARNWARE - try: learnware_model = learnware.get_model() # Check input shape @@ -117,10 +116,9 @@ class EasyStatChecker(BaseChecker): elif spec_type == "RKMEImageSpecification": inputs = np.random.randint(0, 255, size=(10, *input_shape)) else: - raise ValueError(f"not supported spec type for spec_type = {spec_type}") + raise ValueError(f"not supported spec type for spec_type = {spec_type}") # Check output - outputs = learnware.predict(inputs) try: outputs = learnware.predict(inputs) except Exception: diff --git a/learnware/specification/regular/table/rkme.py b/learnware/specification/regular/table/rkme.py index 8f147b2..2b00c61 100644 --- a/learnware/specification/regular/table/rkme.py +++ b/learnware/specification/regular/table/rkme.py @@ -129,7 +129,7 @@ class RKMETableSpecification(RegularStatsSpecification): return # Initialize Z by clustering, utiliing kmeans or faiss to speed up the process. - self._init_z_by_kmeans(X, K) + self._init_z_by_faiss(X, K) self._update_beta(X, nonnegative_beta) # Alternating optimize Z and beta From 68f6fefbc8d0a3841e6b9326bd1d0278e7beddfe Mon Sep 17 00:00:00 2001 From: nju-xy <1582857295@qq.com> Date: Thu, 9 Nov 2023 23:21:27 +0800 Subject: [PATCH 05/10] [FIX] fix pfs --- examples/dataset_pfs_workflow/main.py | 9 +-- learnware/specification/regular/table/rkme.py | 71 +++++++++++-------- 2 files changed, 48 insertions(+), 32 deletions(-) diff --git a/examples/dataset_pfs_workflow/main.py b/examples/dataset_pfs_workflow/main.py index 1f8c348..27e7983 100644 --- a/examples/dataset_pfs_workflow/main.py +++ b/examples/dataset_pfs_workflow/main.py @@ -55,7 +55,7 @@ class PFSDatasetWorkflow: pfs = Dataloader() pfs.regenerate_data() - algo_list = ["ridge"] # , "lgb" + algo_list = ["ridge"] # "ridge", "lgb" for algo in algo_list: pfs.set_algo(algo) pfs.retrain_models() @@ -76,7 +76,8 @@ class PFSDatasetWorkflow: semantic_spec = semantic_specs[0] semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) - easy_market.add_learnware(zip_path, semantic_spec) + x = easy_market.add_learnware(zip_path, semantic_spec) + print(x) print("Total Item:", len(easy_market)) # curr_inds = easy_market._get_ids() @@ -132,7 +133,7 @@ class PFSDatasetWorkflow: rmtree(dir_path) def test(self, regenerate_flag=False): - self.prepare_learnware(regenerate_flag) + # self.prepare_learnware(regenerate_flag) self._init_learnware_market() easy_market = instantiate_learnware_market(market_id="pfs", name="easy") @@ -170,7 +171,7 @@ class PFSDatasetWorkflow: pred_y = learnware.predict(test_x) loss_list.append(pfs.score(test_y, pred_y)) print( - f"Top1-score: {sorted_score_list[0]}, learnware_id: {single_learnware_list[0].id}, loss: {loss_list[0]}" + f"Top1-score: {sorted_score_list[0]}, learnware_id: {single_learnware_list[0].id}, loss: {loss_list[0]}, random: {np.mean(loss_list)}" ) mixture_id = " ".join([learnware.id for learnware in mixture_learnware_list]) diff --git a/learnware/specification/regular/table/rkme.py b/learnware/specification/regular/table/rkme.py index 2b00c61..158a9b4 100644 --- a/learnware/specification/regular/table/rkme.py +++ b/learnware/specification/regular/table/rkme.py @@ -15,13 +15,13 @@ from typing import Tuple, Any, List, Union, Dict import scipy from sklearn.cluster import MiniBatchKMeans -try: - import faiss +# try: +# import faiss - ver = faiss.__version__ - _FAISS_INSTALLED = ver >= "1.7.1" -except ImportError: - _FAISS_INSTALLED = False +# ver = faiss.__version__ +# _FAISS_INSTALLED = ver >= "1.7.1" +# except ImportError: +# _FAISS_INSTALLED = False from ..base import RegularStatsSpecification from ....logger import get_module_logger @@ -129,7 +129,7 @@ class RKMETableSpecification(RegularStatsSpecification): return # Initialize Z by clustering, utiliing kmeans or faiss to speed up the process. - self._init_z_by_faiss(X, K) + self._init_z_by_kmeans(X, K) self._update_beta(X, nonnegative_beta) # Alternating optimize Z and beta @@ -140,22 +140,22 @@ class RKMETableSpecification(RegularStatsSpecification): # Reshape to original dimensions self.z = self.z.reshape(Z_shape) - def _init_z_by_faiss(self, X: Union[np.ndarray, torch.tensor], K: int): - """Intialize Z by faiss clustering. - - Parameters - ---------- - X : np.ndarray or torch.tensor - Raw data in np.ndarray format or torch.tensor format. - K : int - Size of the construced reduced set. - """ - X = X.astype("float32") - numDim = X.shape[1] - kmeans = faiss.Kmeans(numDim, K, niter=100, verbose=False) - kmeans.train(X) - center = torch.from_numpy(kmeans.centroids).double() - self.z = center + # def _init_z_by_faiss(self, X: Union[np.ndarray, torch.tensor], K: int): + # """Intialize Z by faiss clustering. + + # Parameters + # ---------- + # X : np.ndarray or torch.tensor + # Raw data in np.ndarray format or torch.tensor format. + # K : int + # Size of the construced reduced set. + # """ + # X = X.astype("float32") + # numDim = X.shape[1] + # kmeans = faiss.Kmeans(numDim, K, niter=100, verbose=False) + # kmeans.train(X) + # center = torch.from_numpy(kmeans.centroids).double() + # self.z = center def _init_z_by_kmeans(self, X: Union[np.ndarray, torch.tensor], K: int): """Intialize Z by kmeans clustering. @@ -168,7 +168,7 @@ class RKMETableSpecification(RegularStatsSpecification): Size of the construced reduced set. """ X = X.astype("float32") - kmeans = MiniBatchKMeans(n_clusters=K, max_iter=100, verbose=False, n_init="auto") + kmeans = MiniBatchKMeans(n_clusters=K, max_iter=100, verbose=False, n_init='auto') kmeans.fit(X) center = torch.from_numpy(kmeans.cluster_centers_).double() self.z = center @@ -578,6 +578,7 @@ def rkme_solve_qp(K: np.ndarray, C: np.ndarray): K = K.cpu().numpy() if torch.is_tensor(C): C = C.cpu().numpy() + n = K.shape[0] P = np.array(K) P = scipy.sparse.csc_matrix(P) @@ -588,11 +589,25 @@ def rkme_solve_qp(K: np.ndarray, C: np.ndarray): A = np.array(np.ones((1, n))) A = scipy.sparse.csc_matrix(A) b = np.array(np.ones((1, 1))) - # sol = solve_qp(P, q, G, h, A, b, solver="clarabel") # Requires the sum of x to be 1 # sol = solver_qp(P, q, G, h, solver="clarabel") # Otherwise problem = Problem(P, q, G, h, A, b) - sol = solve_problem(problem, solver="clarabel") - w = sol.x + solution = solve_problem(problem, solver="clarabel") + w = solution.x w = torch.from_numpy(w).reshape(-1) - return w, sol.obj + return w, solution.obj + + # from cvxopt import solvers, matrix + # n = K.shape[0] + # P = matrix(K) + # q = matrix(-C) + # G = matrix(-np.eye(n)) + # h = matrix(np.zeros((n, 1))) + # A = matrix(np.ones((1, n))) + # b = matrix(np.ones((1, 1))) + # solvers.options["show_progress"] = False + # sol = solvers.qp(P, q, G, h, A, b) # Requires the sum of x to be 1 + # # sol = solvers.qp(P, q, G, h) # Otherwise + # w = np.array(sol["x"]) + # w = torch.from_numpy(w).reshape(-1) + # return w, sol["primal objective"] From d64365c41d55586f3ec105c7d012ed2f7884ecaf Mon Sep 17 00:00:00 2001 From: nju-xy <1582857295@qq.com> Date: Thu, 9 Nov 2023 23:34:53 +0800 Subject: [PATCH 06/10] [MNT] a little --- examples/dataset_pfs_workflow/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/dataset_pfs_workflow/main.py b/examples/dataset_pfs_workflow/main.py index 27e7983..88cbfb9 100644 --- a/examples/dataset_pfs_workflow/main.py +++ b/examples/dataset_pfs_workflow/main.py @@ -133,7 +133,7 @@ class PFSDatasetWorkflow: rmtree(dir_path) def test(self, regenerate_flag=False): - # self.prepare_learnware(regenerate_flag) + self.prepare_learnware(regenerate_flag) self._init_learnware_market() easy_market = instantiate_learnware_market(market_id="pfs", name="easy") From 032dc4c5e94b04d8a1557e64332c55e15e36cef5 Mon Sep 17 00:00:00 2001 From: nju-xy <1582857295@qq.com> Date: Sat, 11 Nov 2023 11:59:26 +0800 Subject: [PATCH 07/10] [FIX] fix a crucial bug in _calculate_intermediate_K_and_C --- examples/dataset_pfs_workflow/main.py | 3 +-- learnware/market/easy/searcher.py | 3 +-- learnware/specification/regular/table/rkme.py | 1 + tests/test_learnware_client/test_check_learnware.py | 1 + 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/dataset_pfs_workflow/main.py b/examples/dataset_pfs_workflow/main.py index 88cbfb9..c50119d 100644 --- a/examples/dataset_pfs_workflow/main.py +++ b/examples/dataset_pfs_workflow/main.py @@ -76,8 +76,7 @@ class PFSDatasetWorkflow: semantic_spec = semantic_specs[0] semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) - x = easy_market.add_learnware(zip_path, semantic_spec) - print(x) + easy_market.add_learnware(zip_path, semantic_spec) print("Total Item:", len(easy_market)) # curr_inds = easy_market._get_ids() diff --git a/learnware/market/easy/searcher.py b/learnware/market/easy/searcher.py index 5fb5671..73bd593 100644 --- a/learnware/market/easy/searcher.py +++ b/learnware/market/easy/searcher.py @@ -279,7 +279,6 @@ class EasyStatSearcher(BaseSearcher): weight, obj = rkme_solve_qp(K, C) weight = weight.double().to(user_rkme.device) score = user_rkme.inner_prod(user_rkme) + 2 * obj - return weight.detach().cpu().numpy().reshape(-1), score def _calculate_intermediate_K_and_C( @@ -311,7 +310,7 @@ class EasyStatSearcher(BaseSearcher): num = intermediate_K.shape[0] - 1 RKME_list = [learnware.specification.get_stat_spec_by_name(self.stat_spec_type) for learnware in learnware_list] for i in range(intermediate_K.shape[0]): - intermediate_K[num, i] = RKME_list[-1].inner_prod(RKME_list[i]) + intermediate_K[num, i] = intermediate_K[i, num] = RKME_list[-1].inner_prod(RKME_list[i]) intermediate_C[num, 0] = user_rkme.inner_prod(RKME_list[-1]) return intermediate_K, intermediate_C diff --git a/learnware/specification/regular/table/rkme.py b/learnware/specification/regular/table/rkme.py index 158a9b4..4cb4444 100644 --- a/learnware/specification/regular/table/rkme.py +++ b/learnware/specification/regular/table/rkme.py @@ -578,6 +578,7 @@ def rkme_solve_qp(K: np.ndarray, C: np.ndarray): K = K.cpu().numpy() if torch.is_tensor(C): C = C.cpu().numpy() + C = C.reshape(-1) n = K.shape[0] P = np.array(K) diff --git a/tests/test_learnware_client/test_check_learnware.py b/tests/test_learnware_client/test_check_learnware.py index 0e6fca6..36f0a81 100644 --- a/tests/test_learnware_client/test_check_learnware.py +++ b/tests/test_learnware_client/test_check_learnware.py @@ -32,3 +32,4 @@ class TestCheckLearnware(unittest.TestCase): if __name__ == "__main__": unittest.main() + From bc98d0a5dc59d4315a4438fbfc20840c3a147cd7 Mon Sep 17 00:00:00 2001 From: nju-xy <1582857295@qq.com> Date: Sat, 11 Nov 2023 16:22:32 +0800 Subject: [PATCH 08/10] [MNT] remove redundant codes --- learnware/specification/regular/table/rkme.py | 50 +------------------ 1 file changed, 1 insertion(+), 49 deletions(-) diff --git a/learnware/specification/regular/table/rkme.py b/learnware/specification/regular/table/rkme.py index 4cb4444..b217e82 100644 --- a/learnware/specification/regular/table/rkme.py +++ b/learnware/specification/regular/table/rkme.py @@ -15,25 +15,11 @@ from typing import Tuple, Any, List, Union, Dict import scipy from sklearn.cluster import MiniBatchKMeans -# try: -# import faiss - -# ver = faiss.__version__ -# _FAISS_INSTALLED = ver >= "1.7.1" -# except ImportError: -# _FAISS_INSTALLED = False - from ..base import RegularStatsSpecification from ....logger import get_module_logger logger = get_module_logger("rkme") -# if not _FAISS_INSTALLED: -# logger.warning( -# "Required faiss version >= 1.7.1 is not detected! Please run 'conda install -c pytorch faiss-cpu' first" -# ) - - class RKMETableSpecification(RegularStatsSpecification): """Reduced Kernel Mean Embedding (RKME) Specification""" @@ -128,7 +114,7 @@ class RKMETableSpecification(RegularStatsSpecification): self.beta = torch.from_numpy(self.beta).double().to(self.device) return - # Initialize Z by clustering, utiliing kmeans or faiss to speed up the process. + # Initialize Z by clustering, utiliing kmeans to speed up the process. self._init_z_by_kmeans(X, K) self._update_beta(X, nonnegative_beta) @@ -140,23 +126,6 @@ class RKMETableSpecification(RegularStatsSpecification): # Reshape to original dimensions self.z = self.z.reshape(Z_shape) - # def _init_z_by_faiss(self, X: Union[np.ndarray, torch.tensor], K: int): - # """Intialize Z by faiss clustering. - - # Parameters - # ---------- - # X : np.ndarray or torch.tensor - # Raw data in np.ndarray format or torch.tensor format. - # K : int - # Size of the construced reduced set. - # """ - # X = X.astype("float32") - # numDim = X.shape[1] - # kmeans = faiss.Kmeans(numDim, K, niter=100, verbose=False) - # kmeans.train(X) - # center = torch.from_numpy(kmeans.centroids).double() - # self.z = center - def _init_z_by_kmeans(self, X: Union[np.ndarray, torch.tensor], K: int): """Intialize Z by kmeans clustering. @@ -590,25 +559,8 @@ def rkme_solve_qp(K: np.ndarray, C: np.ndarray): A = np.array(np.ones((1, n))) A = scipy.sparse.csc_matrix(A) b = np.array(np.ones((1, 1))) - # sol = solve_qp(P, q, G, h, A, b, solver="clarabel") # Requires the sum of x to be 1 - # sol = solver_qp(P, q, G, h, solver="clarabel") # Otherwise problem = Problem(P, q, G, h, A, b) solution = solve_problem(problem, solver="clarabel") w = solution.x w = torch.from_numpy(w).reshape(-1) return w, solution.obj - - # from cvxopt import solvers, matrix - # n = K.shape[0] - # P = matrix(K) - # q = matrix(-C) - # G = matrix(-np.eye(n)) - # h = matrix(np.zeros((n, 1))) - # A = matrix(np.ones((1, n))) - # b = matrix(np.ones((1, 1))) - # solvers.options["show_progress"] = False - # sol = solvers.qp(P, q, G, h, A, b) # Requires the sum of x to be 1 - # # sol = solvers.qp(P, q, G, h) # Otherwise - # w = np.array(sol["x"]) - # w = torch.from_numpy(w).reshape(-1) - # return w, sol["primal objective"] From 20f64177b6668aa9f251114b750a614919619b7a Mon Sep 17 00:00:00 2001 From: Gene Date: Sat, 11 Nov 2023 21:06:14 +0800 Subject: [PATCH 09/10] [FIX] remove --no-dependencies --- learnware/client/container.py | 1 - learnware/client/utils.py | 1 - 2 files changed, 2 deletions(-) diff --git a/learnware/client/container.py b/learnware/client/container.py index 31a2ab1..25ac521 100644 --- a/learnware/client/container.py +++ b/learnware/client/container.py @@ -340,7 +340,6 @@ class ModelDockerContainer(ModelContainer): "install", "-r", f"{requirements_path_filter}", - "--no-dependencies", ] ) ) diff --git a/learnware/client/utils.py b/learnware/client/utils.py index 09e5142..6a15d65 100644 --- a/learnware/client/utils.py +++ b/learnware/client/utils.py @@ -81,7 +81,6 @@ def install_environment(learnware_dirpath, conda_env): "install", "-r", f"{requirements_path_filter}", - "--no-dependencies", ] ) else: From edf6b76bf5bbdc7229af1997ddc28441d7b53489 Mon Sep 17 00:00:00 2001 From: Gene Date: Sat, 11 Nov 2023 23:02:27 +0800 Subject: [PATCH 10/10] [MNT] modify details and format code --- .../workflows/install_learnware_with_pip.yaml | 5 ----- .../workflows/install_learnware_with_source.yaml | 5 ----- examples/dataset_m5_workflow/main.py | 4 +--- examples/dataset_pfs_workflow/main.py | 4 +--- examples/dataset_text_workflow2/get_data.py | 10 ++++++---- examples/dataset_text_workflow2/main.py | 4 ++-- examples/dataset_text_workflow2/utils.py | 16 ++++++++-------- learnware/market/easy/checker.py | 2 +- learnware/specification/regular/table/rkme.py | 5 ++--- setup.py | 9 --------- .../test_check_learnware.py | 1 - 11 files changed, 21 insertions(+), 44 deletions(-) diff --git a/.github/workflows/install_learnware_with_pip.yaml b/.github/workflows/install_learnware_with_pip.yaml index e5e9ed2..350ed1d 100644 --- a/.github/workflows/install_learnware_with_pip.yaml +++ b/.github/workflows/install_learnware_with_pip.yaml @@ -39,11 +39,6 @@ jobs: conda run -n learnware python -m pip install --upgrade pip conda run -n learnware python -m pip install pytest - - name: Install faiss for MacOS - if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }} - run: | - conda run -n learnware conda install -c pytorch faiss - - name: Install learnware run: | conda run -n learnware python -m pip install learnware diff --git a/.github/workflows/install_learnware_with_source.yaml b/.github/workflows/install_learnware_with_source.yaml index 1702f6b..02cba9e 100644 --- a/.github/workflows/install_learnware_with_source.yaml +++ b/.github/workflows/install_learnware_with_source.yaml @@ -44,11 +44,6 @@ jobs: # stop the build if there are Python syntax errors or undefined names conda run -n learnware python -m flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - - name: Install faiss for MacOS - if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }} - run: | - conda run -n learnware conda install -c pytorch faiss - - name: Install learnware run: | conda run -n learnware python -m pip install . diff --git a/examples/dataset_m5_workflow/main.py b/examples/dataset_m5_workflow/main.py index 2e126e0..bc8a369 100644 --- a/examples/dataset_m5_workflow/main.py +++ b/examples/dataset_m5_workflow/main.py @@ -8,7 +8,6 @@ from shutil import copyfile, rmtree import learnware from learnware.market import instantiate_learnware_market, BaseUserInfo -# from learnware.market import database_ops from learnware.reuse import JobSelectorReuser, AveragingReuser from learnware.specification import generate_rkme_spec from m5 import DataLoader @@ -51,6 +50,7 @@ user_semantic = { "Output": output_description, } + class M5DatasetWorkflow: def _init_m5_dataset(self): m5 = DataLoader() @@ -82,8 +82,6 @@ class M5DatasetWorkflow: easy_market.add_learnware(zip_path, semantic_spec) print("Total Item:", len(easy_market)) - # curr_inds = easy_market._get_ids() - # print("Available ids:", curr_inds) def prepare_learnware(self, regenerate_flag=False): if regenerate_flag: diff --git a/examples/dataset_pfs_workflow/main.py b/examples/dataset_pfs_workflow/main.py index c50119d..e0a8fac 100644 --- a/examples/dataset_pfs_workflow/main.py +++ b/examples/dataset_pfs_workflow/main.py @@ -55,7 +55,7 @@ class PFSDatasetWorkflow: pfs = Dataloader() pfs.regenerate_data() - algo_list = ["ridge"] # "ridge", "lgb" + algo_list = ["ridge"] # "ridge", "lgb" for algo in algo_list: pfs.set_algo(algo) pfs.retrain_models() @@ -79,8 +79,6 @@ class PFSDatasetWorkflow: easy_market.add_learnware(zip_path, semantic_spec) print("Total Item:", len(easy_market)) - # curr_inds = easy_market._get_ids() - # print("Available ids:", curr_inds) def prepare_learnware(self, regenerate_flag=False): if regenerate_flag: diff --git a/examples/dataset_text_workflow2/get_data.py b/examples/dataset_text_workflow2/get_data.py index 216e193..a55e8d7 100644 --- a/examples/dataset_text_workflow2/get_data.py +++ b/examples/dataset_text_workflow2/get_data.py @@ -8,7 +8,9 @@ def get_data(data_root="./data"): dtest = pd.read_csv(os.path.join(data_root, "test.csv")) # returned X(DataFrame), y(Series) - return (dtrain[['discourse_text', 'discourse_type']], - dtrain["discourse_effectiveness"], - dtest[['discourse_text', 'discourse_type']], - dtest["discourse_effectiveness"]) + return ( + dtrain[["discourse_text", "discourse_type"]], + dtrain["discourse_effectiveness"], + dtest[["discourse_text", "discourse_type"]], + dtest["discourse_effectiveness"], + ) diff --git a/examples/dataset_text_workflow2/main.py b/examples/dataset_text_workflow2/main.py index 69765a6..5b3ac96 100644 --- a/examples/dataset_text_workflow2/main.py +++ b/examples/dataset_text_workflow2/main.py @@ -78,10 +78,10 @@ def prepare_model(): modelv_save_path = os.path.join(model_save_root, "uploader_v_%d.pth" % (i)) modell_save_path = os.path.join(model_save_root, "uploader_l_%d.pth" % (i)) - with open(modelv_save_path, 'wb') as f: + with open(modelv_save_path, "wb") as f: pickle.dump(vectorizer, f) - with open(modell_save_path, 'wb') as f: + with open(modell_save_path, "wb") as f: pickle.dump(lgbm, f) logger.info("Model saved to '%s' and '%s'" % (modelv_save_path, modell_save_path)) diff --git a/examples/dataset_text_workflow2/utils.py b/examples/dataset_text_workflow2/utils.py index 4726407..247f706 100644 --- a/examples/dataset_text_workflow2/utils.py +++ b/examples/dataset_text_workflow2/utils.py @@ -39,11 +39,11 @@ def generate_uploader(data_x: pd.Series, data_y: pd.Series, n_uploaders=50, data return os.makedirs(data_save_root, exist_ok=True) - types = data_x['discourse_type'].unique() + types = data_x["discourse_type"].unique() for i in range(n_uploaders): - indices = data_x['discourse_type'] == types[i] - selected_X = data_x[indices]['discourse_text'].to_list() + indices = data_x["discourse_type"] == types[i] + selected_X = data_x[indices]["discourse_text"].to_list() selected_y = data_y[indices].to_list() X_save_dir = os.path.join(data_save_root, "uploader_%d_X.pkl" % (i)) @@ -61,11 +61,11 @@ def generate_user(data_x, data_y, n_users=50, data_save_root=None): return os.makedirs(data_save_root, exist_ok=True) - types = data_x['discourse_type'].unique() + types = data_x["discourse_type"].unique() for i in range(n_users): - indices = data_x['discourse_type'] == types[i] - selected_X = data_x[indices]['discourse_text'].to_list() + indices = data_x["discourse_type"] == types[i] + selected_X = data_x[indices]["discourse_text"].to_list() selected_y = data_y[indices].to_list() X_save_dir = os.path.join(data_save_root, "user_%d_X.pkl" % (i)) @@ -80,10 +80,10 @@ def generate_user(data_x, data_y, n_users=50, data_save_root=None): # Train Uploaders' models def train(X, y, out_classes): - vectorizer = TfidfVectorizer(stop_words='english') + vectorizer = TfidfVectorizer(stop_words="english") X_tfidf = vectorizer.fit_transform(X) - lgbm = LGBMClassifier(boosting_type='dart', n_estimators=500, num_leaves=21) + lgbm = LGBMClassifier(boosting_type="dart", n_estimators=500, num_leaves=21) lgbm.fit(X_tfidf, y) return vectorizer, lgbm diff --git a/learnware/market/easy/checker.py b/learnware/market/easy/checker.py index d0d50cf..6419b98 100644 --- a/learnware/market/easy/checker.py +++ b/learnware/market/easy/checker.py @@ -116,7 +116,7 @@ class EasyStatChecker(BaseChecker): elif spec_type == "RKMEImageSpecification": inputs = np.random.randint(0, 255, size=(10, *input_shape)) else: - raise ValueError(f"not supported spec type for spec_type = {spec_type}") + raise ValueError(f"not supported spec type for spec_type = {spec_type}") # Check output try: diff --git a/learnware/specification/regular/table/rkme.py b/learnware/specification/regular/table/rkme.py index b217e82..a37d509 100644 --- a/learnware/specification/regular/table/rkme.py +++ b/learnware/specification/regular/table/rkme.py @@ -1,14 +1,12 @@ from __future__ import annotations import os - import copy import torch import json import codecs import random import numpy as np -# from cvxopt import solvers, matrix from qpsolvers import solve_qp, Problem, solve_problem from collections import Counter from typing import Tuple, Any, List, Union, Dict @@ -20,6 +18,7 @@ from ....logger import get_module_logger logger = get_module_logger("rkme") + class RKMETableSpecification(RegularStatsSpecification): """Reduced Kernel Mean Embedding (RKME) Specification""" @@ -137,7 +136,7 @@ class RKMETableSpecification(RegularStatsSpecification): Size of the construced reduced set. """ X = X.astype("float32") - kmeans = MiniBatchKMeans(n_clusters=K, max_iter=100, verbose=False, n_init='auto') + kmeans = MiniBatchKMeans(n_clusters=K, max_iter=100, verbose=False, n_init="auto") kmeans.fit(X) center = torch.from_numpy(kmeans.cluster_centers_).double() self.z = center diff --git a/setup.py b/setup.py index 6d2777e..2c0df29 100644 --- a/setup.py +++ b/setup.py @@ -54,12 +54,6 @@ REQUIRED = [ "numpy>=1.20.0", "pandas>=0.25.1", "scipy>=1.0.0", -<<<<<<< HEAD - "matplotlib>=3.1.3", - "torch>=1.11.0", -======= - "cvxopt>=1.3.0", ->>>>>>> 93df27b2a16a169ecfb93e3a2e149b6c1ea56902 "tqdm>=4.65.0", "scikit-learn>=0.22", "joblib>=1.2.0", @@ -76,9 +70,6 @@ REQUIRED = [ "qpsolvers[clarabel]>=4.0.1", ] -# if get_platform() != MACOS: -# REQUIRED.append("faiss-cpu>=1.7.1") - here = os.path.abspath(os.path.dirname(__file__)) with open(os.path.join(here, "README.md"), encoding="utf-8") as f: long_description = f.read() diff --git a/tests/test_learnware_client/test_check_learnware.py b/tests/test_learnware_client/test_check_learnware.py index 36f0a81..0e6fca6 100644 --- a/tests/test_learnware_client/test_check_learnware.py +++ b/tests/test_learnware_client/test_check_learnware.py @@ -32,4 +32,3 @@ class TestCheckLearnware(unittest.TestCase): if __name__ == "__main__": unittest.main() -