From fb216a16070a6cbc6ecd40345c9ced070f7259ac Mon Sep 17 00:00:00 2001 From: nju-xy <1582857295@qq.com> Date: Tue, 14 Nov 2023 11:21:00 +0800 Subject: [PATCH] [MNT] add requirements for text_workflow2 --- .../pfs/pfs_cross_transfer.py | 4 +++- .../example_files/example_init.py | 2 +- .../example_files/requirements.txt | 4 ++++ examples/dataset_text_workflow2/main.py | 21 +++++++++++++++---- learnware/market/easy/checker.py | 4 ++-- 5 files changed, 27 insertions(+), 8 deletions(-) create mode 100644 examples/dataset_text_workflow2/example_files/requirements.txt diff --git a/examples/dataset_pfs_workflow/pfs/pfs_cross_transfer.py b/examples/dataset_pfs_workflow/pfs/pfs_cross_transfer.py index 93a3fa3..5f69127 100644 --- a/examples/dataset_pfs_workflow/pfs/pfs_cross_transfer.py +++ b/examples/dataset_pfs_workflow/pfs/pfs_cross_transfer.py @@ -85,7 +85,9 @@ def get_split_errs(algo): split = train_xs.shape[0] - proportion_list[tmp] model.fit( - train_xs[split:,], + train_xs[ + split:, + ], train_ys[split:], eval_set=[(val_xs, val_ys)], early_stopping_rounds=50, diff --git a/examples/dataset_text_workflow/example_files/example_init.py b/examples/dataset_text_workflow/example_files/example_init.py index 0052925..f15d335 100644 --- a/examples/dataset_text_workflow/example_files/example_init.py +++ b/examples/dataset_text_workflow/example_files/example_init.py @@ -19,7 +19,7 @@ class Model(BaseModel): input_dim = 768 classifier_head = RobertaClassificationHead(num_classes=num_classes, input_dim=input_dim) self.model = XLMR_BASE_ENCODER.get_model(head=classifier_head).to(self.device) - self.model.load_state_dict(torch.load(os.path.join(dir_path, "model.pth"), map_location=torch.device('cpu'))) + self.model.load_state_dict(torch.load(os.path.join(dir_path, "model.pth"), map_location=torch.device("cpu"))) def fit(self, X: np.ndarray, y: np.ndarray): pass diff --git a/examples/dataset_text_workflow2/example_files/requirements.txt b/examples/dataset_text_workflow2/example_files/requirements.txt new file mode 100644 index 0000000..da986b1 --- /dev/null +++ b/examples/dataset_text_workflow2/example_files/requirements.txt @@ -0,0 +1,4 @@ +numpy +pickle +lightgbm +scikit-learn \ No newline at end of file diff --git a/examples/dataset_text_workflow2/main.py b/examples/dataset_text_workflow2/main.py index 5b3ac96..c559476 100644 --- a/examples/dataset_text_workflow2/main.py +++ b/examples/dataset_text_workflow2/main.py @@ -87,7 +87,9 @@ def prepare_model(): logger.info("Model saved to '%s' and '%s'" % (modelv_save_path, modell_save_path)) -def prepare_learnware(data_path, modelv_path, modell_path, init_file_path, yaml_path, save_root, zip_name): +def prepare_learnware( + data_path, modelv_path, modell_path, init_file_path, yaml_path, env_file_path, save_root, zip_name +): os.makedirs(save_root, exist_ok=True) tmp_spec_path = os.path.join(save_root, "rkme.json") @@ -96,6 +98,7 @@ def prepare_learnware(data_path, modelv_path, modell_path, init_file_path, yaml_ tmp_yaml_path = os.path.join(save_root, "learnware.yaml") tmp_init_path = os.path.join(save_root, "__init__.py") + tmp_env_path = os.path.join(save_root, "requirements.txt") with open(data_path, "rb") as f: X = pickle.load(f) @@ -115,6 +118,7 @@ def prepare_learnware(data_path, modelv_path, modell_path, init_file_path, yaml_ copyfile(yaml_path, tmp_yaml_path) copyfile(init_file_path, tmp_init_path) + copyfile(env_file_path, tmp_env_path) zip_file_name = os.path.join(learnware_pool_dir, "%s.zip" % (zip_name)) with zipfile.ZipFile(zip_file_name, "w", compression=zipfile.ZIP_DEFLATED) as zip_obj: zip_obj.write(tmp_spec_path, "rkme.json") @@ -124,6 +128,7 @@ def prepare_learnware(data_path, modelv_path, modell_path, init_file_path, yaml_ zip_obj.write(tmp_yaml_path, "learnware.yaml") zip_obj.write(tmp_init_path, "__init__.py") + zip_obj.write(tmp_env_path, "requirements.txt") rmtree(save_root) logger.info("New Learnware Saved to %s" % (zip_file_name)) return zip_file_name @@ -144,8 +149,16 @@ def prepare_market(): init_file_path = "./example_files/example_init.py" yaml_file_path = "./example_files/example_yaml.yaml" + env_file_path = "./example_files/requirements.txt" new_learnware_path = prepare_learnware( - data_path, modelv_path, modell_path, init_file_path, yaml_file_path, tmp_dir, "%s_%d" % (dataset, i) + data_path, + modelv_path, + modell_path, + init_file_path, + yaml_file_path, + env_file_path, + tmp_dir, + "%s_%d" % (dataset, i), ) semantic_spec = semantic_specs[0] semantic_spec["Name"]["Values"] = "learnware_%d" % (i) @@ -239,6 +252,6 @@ def test_search(load_market=True): if __name__ == "__main__": - prepare_data() - prepare_model() + # prepare_data() + # prepare_model() test_search(load_market=False) diff --git a/learnware/market/easy/checker.py b/learnware/market/easy/checker.py index 54aa0e5..c4e3ef9 100644 --- a/learnware/market/easy/checker.py +++ b/learnware/market/easy/checker.py @@ -51,7 +51,7 @@ class EasySemanticChecker(BaseChecker): assert int(k) >= 0 and int(k) < dim, f"Dimension number in [0, {dim})" assert isinstance(v, str), "Description must be string" - return EasySemanticChecker.NONUSABLE_LEARNWARE, 'EasySemanticChecker Success' + return EasySemanticChecker.NONUSABLE_LEARNWARE, "EasySemanticChecker Success" except AssertionError as err: logger.warning(f"semantic_specification is not valid due to {err}!") @@ -128,7 +128,7 @@ class EasyStatChecker(BaseChecker): except Exception: message = f"The learnware {learnware.id} prediction is not avaliable!" logger.warning(message) - message += '\r\n' + traceback.format_exc() + message += "\r\n" + traceback.format_exc() return self.INVALID_LEARNWARE, message if semantic_spec["Task"]["Values"][0] in ("Classification", "Regression"):