| @@ -50,6 +50,7 @@ semantic_specs = [ | |||
| "Description": {"Values": "", "Type": "String"}, | |||
| "Name": {"Values": "learnware_1", "Type": "String"}, | |||
| "Output": {"Dimension": 10}, | |||
| "License": {"Values": ["MIT"], "Type": "Class"}, | |||
| } | |||
| ] | |||
| @@ -60,6 +61,7 @@ user_semantic = { | |||
| "Scenario": {"Values": ["Business"], "Type": "Tag"}, | |||
| "Description": {"Values": "", "Type": "String"}, | |||
| "Name": {"Values": "", "Type": "String"}, | |||
| "License": {"Values": ["MIT"], "Type": "Class"}, | |||
| } | |||
| @@ -175,7 +177,10 @@ def test_search(gamma=0.1, load_market=True): | |||
| pred_y = single_item.learnware.predict(user_data) | |||
| acc = eval_prediction(pred_y, user_label) | |||
| acc_list.append(acc) | |||
| logger.info("Search rank: %d, score: %.3f, learnware_id: %s, acc: %.3f" % (idx, single_item.score, single_item.learnware.id, acc)) | |||
| logger.info( | |||
| "Search rank: %d, score: %.3f, learnware_id: %s, acc: %.3f" | |||
| % (idx, single_item.score, single_item.learnware.id, acc) | |||
| ) | |||
| # test reuse (job selector) | |||
| # reuse_baseline = JobSelectorReuser(learnware_list=mixture_learnware_list, herding_num=100) | |||
| @@ -36,6 +36,7 @@ semantic_specs = [ | |||
| "Name": {"Values": "learnware_1", "Type": "String"}, | |||
| "Input": input_description, | |||
| "Output": output_description, | |||
| "License": {"Values": ["MIT"], "Type": "Class"}, | |||
| } | |||
| ] | |||
| @@ -48,6 +49,7 @@ user_semantic = { | |||
| "Name": {"Values": "", "Type": "String"}, | |||
| "Input": input_description, | |||
| "Output": output_description, | |||
| "License": {"Values": ["MIT"], "Type": "Class"}, | |||
| } | |||
| @@ -158,7 +160,7 @@ class M5DatasetWorkflow: | |||
| search_result = easy_market.search_learnware(user_info) | |||
| single_result = search_result.get_single_results() | |||
| multiple_result = search_result.get_multiple_results() | |||
| print(f"search result of user{idx}:") | |||
| print( | |||
| f"single model num: {len(single_result)}, max_score: {single_result[0].score}, min_score: {single_result[-1].score}" | |||
| @@ -66,6 +66,7 @@ def main(): | |||
| "Scenario": {"Values": ["Business"], "Type": "Tag"}, | |||
| "Description": {"Values": "A sales-forecasting model from Walmart store", "Type": "String"}, | |||
| "Name": {"Values": name, "Type": "String"}, | |||
| "License": {"Values": ["MIT"], "Type": "Class"}, | |||
| } | |||
| res = session.post( | |||
| submit_url, | |||
| @@ -35,6 +35,7 @@ semantic_specs = [ | |||
| "Name": {"Values": "learnware_1", "Type": "String"}, | |||
| "Input": input_description, | |||
| "Output": output_description, | |||
| "License": {"Values": ["MIT"], "Type": "Class"}, | |||
| } | |||
| ] | |||
| @@ -47,6 +48,7 @@ user_semantic = { | |||
| "Name": {"Values": "", "Type": "String"}, | |||
| "Input": input_description, | |||
| "Output": output_description, | |||
| "License": {"Values": ["MIT"], "Type": "Class"}, | |||
| } | |||
| @@ -155,7 +157,7 @@ class PFSDatasetWorkflow: | |||
| search_result = easy_market.search_learnware(user_info) | |||
| single_result = search_result.get_single_results() | |||
| multiple_result = search_result.get_multiple_results() | |||
| print(f"search result of user{idx}:") | |||
| print( | |||
| f"single model num: {len(single_result)}, max_score: {single_result[0].score}, min_score: {single_result[-1].score}" | |||
| @@ -69,6 +69,7 @@ def main(): | |||
| "Type": "String", | |||
| }, | |||
| "Name": {"Values": name, "Type": "String"}, | |||
| "License": {"Values": ["MIT"], "Type": "Class"}, | |||
| } | |||
| res = session.post( | |||
| submit_url, | |||
| @@ -48,6 +48,7 @@ semantic_specs = [ | |||
| "Description": {"Values": "", "Type": "String"}, | |||
| "Name": {"Values": "learnware_1", "Type": "String"}, | |||
| "Output": output_description, | |||
| "License": {"Values": ["MIT"], "Type": "Class"}, | |||
| } | |||
| ] | |||
| @@ -59,6 +60,7 @@ user_semantic = { | |||
| "Description": {"Values": "", "Type": "String"}, | |||
| "Name": {"Values": "", "Type": "String"}, | |||
| "Output": output_description, | |||
| "License": {"Values": ["MIT"], "Type": "Class"}, | |||
| } | |||
| @@ -73,7 +75,6 @@ class TextDatasetWorkflow: | |||
| generate_uploader(X_train, y_train, n_uploaders=n_uploaders, data_save_root=uploader_save_root) | |||
| generate_user(X_test, y_test, n_users=n_users, data_save_root=user_save_root) | |||
| def _prepare_model(self): | |||
| dataloader = TextDataLoader(data_save_root, train=True) | |||
| for i in range(n_uploaders): | |||
| @@ -92,9 +93,8 @@ class TextDatasetWorkflow: | |||
| logger.info("Model saved to '%s' and '%s'" % (modelv_save_path, modell_save_path)) | |||
| def _prepare_learnware(self, | |||
| data_path, modelv_path, modell_path, init_file_path, yaml_path, env_file_path, save_root, zip_name | |||
| def _prepare_learnware( | |||
| self, data_path, modelv_path, modell_path, init_file_path, yaml_path, env_file_path, save_root, zip_name | |||
| ): | |||
| os.makedirs(save_root, exist_ok=True) | |||
| tmp_spec_path = os.path.join(save_root, "rkme.json") | |||
| @@ -139,7 +139,6 @@ class TextDatasetWorkflow: | |||
| logger.info("New Learnware Saved to %s" % (zip_file_name)) | |||
| return zip_file_name | |||
| def prepare_market(self, regenerate_flag=False): | |||
| if regenerate_flag: | |||
| self._init_text_dataset() | |||
| @@ -175,7 +174,6 @@ class TextDatasetWorkflow: | |||
| logger.info("Total Item: %d" % (len(text_market))) | |||
| def test(self, regenerate_flag=False): | |||
| self.prepare_market(regenerate_flag) | |||
| text_market = instantiate_learnware_market(market_id="ae") | |||
| @@ -199,11 +197,11 @@ class TextDatasetWorkflow: | |||
| user_stat_spec.generate_stat_spec_from_data(X=user_data) | |||
| user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETextSpecification": user_stat_spec}) | |||
| logger.info("Searching Market for user: %d" % (i)) | |||
| search_result = text_market.search_learnware(user_info) | |||
| single_result = search_result.get_single_results() | |||
| multiple_result = search_result.get_multiple_results() | |||
| print(f"search result of user{i}:") | |||
| print( | |||
| f"single model num: {len(single_result)}, max_score: {single_result[0].score}, min_score: {single_result[-1].score}" | |||
| @@ -220,7 +218,7 @@ class TextDatasetWorkflow: | |||
| print( | |||
| f"Top1-score: {single_result[0].score}, learnware_id: {single_result[0].learnware.id}, acc: {acc_list[0]}" | |||
| ) | |||
| if len(multiple_result) > 0: | |||
| mixture_id = " ".join([learnware.id for learnware in multiple_result[0].learnwares]) | |||
| print(f"mixture_score: {multiple_result[0].score}, mixture_learnware: {mixture_id}") | |||
| @@ -35,6 +35,7 @@ user_semantic = { | |||
| "Scenario": {"Values": ["Education"], "Type": "Tag"}, | |||
| "Description": {"Values": "", "Type": "String"}, | |||
| "Name": {"Values": "", "Type": "String"}, | |||
| "License": {"Values": ["MIT"], "Type": "Class"}, | |||
| } | |||
| @@ -259,15 +260,15 @@ class TestMarket(unittest.TestCase): | |||
| str(key): semantic_spec["Input"]["Description"][str(key)] for key in range(user_dim) | |||
| } | |||
| user_info = BaseUserInfo(semantic_spec=semantic_spec, stat_info={"RKMETableSpecification": user_spec}) | |||
| search_result = hetero_market.search_learnware(user_info) | |||
| single_result = search_result.get_single_results() | |||
| multiple_result = search_result.get_multiple_results() | |||
| print(f"search result of user{idx}:") | |||
| for single_item in single_result: | |||
| print(f"score: {single_item.score}, learnware_id: {single_item.learnware.id}") | |||
| for multiple_item in multiple_result: | |||
| print( | |||
| f"mixture_score: {multiple_item.score}, mixture_learnware_ids: {[item.id for item in multiple_item.learnwares]}" | |||
| @@ -335,7 +336,7 @@ class TestMarket(unittest.TestCase): | |||
| print(f"search result of user{idx}:") | |||
| for single_item in single_result: | |||
| print(f"score: {single_item.score}, learnware_id: {single_item.learnware.id}") | |||
| for multiple_item in multiple_result: | |||
| print(f"mixture_score: {multiple_item.score}\n") | |||
| mixture_id = " ".join([learnware.id for learnware in multiple_item.learnwares]) | |||
| @@ -363,9 +364,11 @@ class TestMarket(unittest.TestCase): | |||
| # print search results | |||
| for single_item in single_result: | |||
| print(f"score: {single_item.score}, learnware_id: {single_item.learnware.id}") | |||
| for multiple_item in multiple_result: | |||
| print(f"mixture_score: {multiple_item.score}, mixture_learnware_ids: {[item.id for item in multiple_item.learnwares]}") | |||
| print( | |||
| f"mixture_score: {multiple_item.score}, mixture_learnware_ids: {[item.id for item in multiple_item.learnwares]}" | |||
| ) | |||
| # single model reuse | |||
| hetero_learnware = HeteroMapAlignLearnware(single_result[0].learnware, mode="regression") | |||