From 1034d0ef7eb4e04ae7967882b09426c567698708 Mon Sep 17 00:00:00 2001 From: zouxiaochuan Date: Sun, 12 Nov 2023 00:46:18 +0800 Subject: [PATCH 1/4] [MNT]: add read learnware info from storage --- learnware/client/package_utils.py | 8 +++++++- learnware/market/easy/database_ops.py | 15 +++++++++++++++ learnware/market/easy/organizer.py | 20 +++++++++++++++++++- 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/learnware/client/package_utils.py b/learnware/client/package_utils.py index 077fcac..cb99c85 100644 --- a/learnware/client/package_utils.py +++ b/learnware/client/package_utils.py @@ -105,7 +105,13 @@ def filter_nonexist_conda_packages(packages: list) -> Tuple[List[str], List[str] command = f"conda env create --name env_test --file {test_yaml_file} --dry-run --json" result = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - output = json.loads(result.stdout.strip()).get("bad_deps", []) + stdout = result.stdout.strip() + last_bracket = stdout.rfind("\n{") + if last_bracket != -1: + stdout = stdout[last_bracket:] + pass + print(stdout) + output = json.loads(stdout).get("bad_deps", []) if len(output) > 0: exist_packages = [] diff --git a/learnware/market/easy/database_ops.py b/learnware/market/easy/database_ops.py index c9fb3de..e128653 100644 --- a/learnware/market/easy/database_ops.py +++ b/learnware/market/easy/database_ops.py @@ -167,6 +167,21 @@ class DatabaseOperations(object): pass pass + def get_learnware_info(self, id: str): + with self.engine.connect() as conn: + r = conn.execute(text("SELECT semantic_spec, zip_path, folder_path, use_flag FROM tb_learnware WHERE id=:id;"), dict(id=id)) + row = r.fetchone() + if row is None: + return None + else: + semantic_spec = json.loads(row[0]) + zip_path = row[1] + folder_path = row[2] + use_flag = int(row[3]) + return {'semantic_spec': semantic_spec, 'zip_path': zip_path, 'folder_path': folder_path, 'use_flag': use_flag} + pass + pass + def load_market(self): with self.engine.connect() as conn: cursor = conn.execute(text("SELECT id, semantic_spec, zip_path, folder_path, use_flag FROM tb_learnware;")) diff --git a/learnware/market/easy/organizer.py b/learnware/market/easy/organizer.py index 9337841..27b974d 100644 --- a/learnware/market/easy/organizer.py +++ b/learnware/market/easy/organizer.py @@ -3,7 +3,7 @@ import copy import zipfile import tempfile from shutil import copyfile, rmtree -from typing import Tuple, List, Union +from typing import Tuple, List, Union, Dict from .database_ops import DatabaseOperations from ..base import BaseOrganizer, BaseChecker @@ -392,5 +392,23 @@ class EasyOrganizer(BaseOrganizer): self.use_flags[learnware_id] = self.dbops.get_learnware_use_flag(learnware_id) pass + def get_learnware_info_from_storage(self, learnware_id: str) -> Dict: + """return learnware zip path and semantic_specification from storage + + Parameters + ---------- + learnware_id : str + learnware id + + Returns + ------- + Dict + - semantic_spec: semantic_specification + - zip_path: zip_path + - folder_path: folder_path + - use_flag: use_flag + """ + return self.dbops.get_learnware_info(learnware_id) + def __len__(self): return len(self.learnware_list) From c7bbef5388cd44313bcd03347eccbea4af2fdb2e Mon Sep 17 00:00:00 2001 From: shihy Date: Sun, 12 Nov 2023 16:00:58 +0800 Subject: [PATCH 2/4] [Fix] Adapting to function rkme_solve_qp updates --- learnware/specification/regular/image/rkme.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/learnware/specification/regular/image/rkme.py b/learnware/specification/regular/image/rkme.py index 4421f91..df7a2a3 100644 --- a/learnware/specification/regular/image/rkme.py +++ b/learnware/specification/regular/image/rkme.py @@ -18,7 +18,7 @@ from tqdm import tqdm from . import cnn_gp from ..base import RegularStatsSpecification -from ..table.rkme import solve_qp, choose_device, setup_seed +from ..table.rkme import rkme_solve_qp, choose_device, setup_seed class RKMEImageSpecification(RegularStatsSpecification): @@ -97,6 +97,11 @@ class RKMEImageSpecification(RegularStatsSpecification): ------- """ + if len(X.shape) != 4: + raise ValueError("X should be in shape of [N, C, {0:d}, {0:d}]. ".format( + RKMEImageSpecification.IMAGE_WIDTH + )) + if ( X.shape[2] != RKMEImageSpecification.IMAGE_WIDTH or X.shape[3] != RKMEImageSpecification.IMAGE_WIDTH ) and not resize: @@ -175,7 +180,7 @@ class RKMEImageSpecification(RegularStatsSpecification): C = torch.sum(C, dim=1) / x_features.shape[0] if nonnegative_beta: - beta = solve_qp(K.double(), C.double()).to(self.device) + beta = rkme_solve_qp(K.double(), C.double())[0].to(self.device) else: beta = torch.linalg.inv(K + torch.eye(K.shape[0]).to(self.device) * 1e-5) @ C From 1d83442b855b04c7ece9cae4d6e053dfbc3082b1 Mon Sep 17 00:00:00 2001 From: shihy Date: Sun, 12 Nov 2023 16:25:33 +0800 Subject: [PATCH 3/4] [MNT] format code by black --- learnware/specification/regular/image/rkme.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/learnware/specification/regular/image/rkme.py b/learnware/specification/regular/image/rkme.py index df7a2a3..75086c2 100644 --- a/learnware/specification/regular/image/rkme.py +++ b/learnware/specification/regular/image/rkme.py @@ -98,9 +98,9 @@ class RKMEImageSpecification(RegularStatsSpecification): """ if len(X.shape) != 4: - raise ValueError("X should be in shape of [N, C, {0:d}, {0:d}]. ".format( - RKMEImageSpecification.IMAGE_WIDTH - )) + raise ValueError( + "X should be in shape of [N, C, {0:d}, {0:d}]. ".format(RKMEImageSpecification.IMAGE_WIDTH) + ) if ( X.shape[2] != RKMEImageSpecification.IMAGE_WIDTH or X.shape[3] != RKMEImageSpecification.IMAGE_WIDTH From 9be1495a2f68fe04d796bb31a9d13432ea4f37fb Mon Sep 17 00:00:00 2001 From: shihy Date: Sun, 12 Nov 2023 16:29:38 +0800 Subject: [PATCH 4/4] [MNT] modify error msg --- learnware/specification/regular/image/rkme.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/learnware/specification/regular/image/rkme.py b/learnware/specification/regular/image/rkme.py index 75086c2..5bf57fd 100644 --- a/learnware/specification/regular/image/rkme.py +++ b/learnware/specification/regular/image/rkme.py @@ -98,9 +98,7 @@ class RKMEImageSpecification(RegularStatsSpecification): """ if len(X.shape) != 4: - raise ValueError( - "X should be in shape of [N, C, {0:d}, {0:d}]. ".format(RKMEImageSpecification.IMAGE_WIDTH) - ) + raise ValueError("X should be in shape of [N, C, H, W]. ") if ( X.shape[2] != RKMEImageSpecification.IMAGE_WIDTH or X.shape[3] != RKMEImageSpecification.IMAGE_WIDTH