beimingwu
/
learnware
Not watched
1
0
Fork 0
Code
Releases 1 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

[MNT] remove pfs and m5 workflow

tags/v0.3.2
Gene 2 years ago
parent
08ba9320d3
commit
c5f9a42e7a
20 changed files with 0 additions and 3011 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +0
    -8
      examples/dataset_m5_workflow/example.yaml
  2. +0
    -21
      examples/dataset_m5_workflow/example_init.py
  3. +0
    -3
      examples/dataset_m5_workflow/m5/README.md
  4. +0
    -65
      examples/dataset_m5_workflow/m5/__init__.py
  5. +0
    -139
      examples/dataset_m5_workflow/m5/config.py
  6. +0
    -338
      examples/dataset_m5_workflow/m5/generate_data.py
  7. +0
    -452
      examples/dataset_m5_workflow/m5/train.py
  8. +0
    -177
      examples/dataset_m5_workflow/m5/utils.py
  9. +0
    -211
      examples/dataset_m5_workflow/main.py
  10. +0
    -87
      examples/dataset_m5_workflow/upload.py
  11. +0
    -8
      examples/dataset_pfs_workflow/example.yaml
  12. +0
    -20
      examples/dataset_pfs_workflow/example_init.py
  13. +0
    -208
      examples/dataset_pfs_workflow/main.py
  14. +0
    -48
      examples/dataset_pfs_workflow/pfs/README.md
  15. +0
    -77
      examples/dataset_pfs_workflow/pfs/__init__.py
  16. +0
    -272
      examples/dataset_pfs_workflow/pfs/config.py
  17. +0
    -21
      examples/dataset_pfs_workflow/pfs/paths.py
  18. +0
    -382
      examples/dataset_pfs_workflow/pfs/pfs_cross_transfer.py
  19. +0
    -384
      examples/dataset_pfs_workflow/pfs/split_data.py
  20. +0
    -90
      examples/dataset_pfs_workflow/upload.py

+ 0
- 8
examples/dataset_m5_workflow/example.yaml View File

@@ -1,8 +0,0 @@
model:
class_name: Model
kwargs: {}
stat_specifications:
- module_path: learnware.specification
class_name: RKMETableSpecification
file_name: rkme.json
kwargs: {}

+ 0
- 21
examples/dataset_m5_workflow/example_init.py View File

@@ -1,21 +0,0 @@
import os
import joblib
import numpy as np
import lightgbm as lgb
from learnware.model import BaseModel


class Model(BaseModel):
def __init__(self):
super(Model, self).__init__(input_shape=(82,), output_shape=(1,))
dir_path = os.path.dirname(os.path.abspath(__file__))
self.model = lgb.Booster(model_file=os.path.join(dir_path, "model.out"))

def fit(self, X: np.ndarray, y: np.ndarray):
pass

def predict(self, X: np.ndarray) -> np.ndarray:
return self.model.predict(X)

def finetune(self, X: np.ndarray, y: np.ndarray):
pass

+ 0
- 3
examples/dataset_m5_workflow/m5/README.md View File

@@ -1,3 +0,0 @@
# M5 Dataset

Walmart store, involves the unit sales of various products sold in the USA, organized in the form of grouped time series. More specifically, the dataset involves the unit sales of 3049 products, classified in 3 product categories (Hobbies, Foods, and Household).

+ 0
- 65
examples/dataset_m5_workflow/m5/__init__.py View File

@@ -1,65 +0,0 @@
from cgi import test
import os
import joblib
import lightgbm as lgb


from .config import store_list, model_dir
from .utils import acquire_data, get_weights, model_predict, score, measure_aux_algo
from .generate_data import regenerate_data
from .train import retrain_models, grid_training_sample, train_adaptation_grid


class DataLoader:
def __init__(self):
self.algo = "ridge"

def set_algo(self, algo):
self.algo = algo

def get_algo_list(self):
return ["lgb", "ridge"]

def get_idx_list(self):
return list(range(len(store_list)))

def get_idx_data(self, idx):
store = store_list[idx]
# fill_flag = self.algo == "ridge"
fill_flag = True
return acquire_data(store, fill_flag)

def get_weights(self):
return get_weights(self.algo)

def get_model_path(self, idx):
return os.path.join(model_dir, "{}_{}.out".format(self.algo, store_list[idx]))

def predict(self, idx, test_x):
store = store_list[idx]

if os.path.exists(os.path.join(model_dir, f"{self.algo}_{store}.out")):
return model_predict(self.algo, idx, test_x)
else:
self.retrain_models()
return model_predict(self.algo, idx, test_x)

def score(self, real_y, pred_y, sample_weight=None, multioutput="raw_values"):
return score(real_y, pred_y, sample_weight, multioutput)

def regenerate_data(self):
regenerate_data()

def retrain_models(self):
retrain_models(self.algo)

def grid_training_sample(self, user_list=list(range(10))):
grid_training_sample(self.algo, user_list)

def train_adaptation_grid(
self, max_sample, test_sample, user_list=list(range(10)), adaptation_model=[], residual=False
):
train_adaptation_grid(self.algo, max_sample, test_sample, user_list, adaptation_model, residual)

def measure_aux_algo(self, idx, test_sample, model):
return measure_aux_algo(idx, test_sample, model)

+ 0
- 139
examples/dataset_m5_workflow/m5/config.py View File

@@ -1,139 +0,0 @@
import os


ROOT_PATH = os.path.abspath(os.path.join(__file__, "..", "data"))
raw_data_dir = os.path.join(ROOT_PATH, "raw")
processed_data_dir = os.path.join(ROOT_PATH, "processed")
model_dir = os.path.join(ROOT_PATH, "models")
grid_dir = os.path.join(ROOT_PATH, "grid_sample")


TARGET = "sales"
START_TRAIN = 1
END_TRAIN = 1941 - 28


category_list = ["item_id", "dept_id", "cat_id", "event_name_1", "event_name_2", "event_type_1", "event_type_2"]
features_columns = [
"item_id",
"dept_id",
"cat_id",
"release",
"sell_price",
"price_max",
"price_min",
"price_std",
"price_mean",
"price_norm",
"price_nunique",
"item_nunique",
"price_momentum",
"price_momentum_m",
"price_momentum_y",
"event_name_1",
"event_type_1",
"event_name_2",
"event_type_2",
"snap",
"tm_d",
"tm_w",
"tm_m",
"tm_y",
"tm_wm",
"tm_dw",
"tm_w_end",
"sales_lag_28",
"sales_lag_29",
"sales_lag_30",
"sales_lag_31",
"sales_lag_32",
"sales_lag_33",
"sales_lag_34",
"sales_lag_35",
"sales_lag_36",
"sales_lag_37",
"sales_lag_38",
"sales_lag_39",
"sales_lag_40",
"sales_lag_41",
"sales_lag_42",
"rolling_mean_7",
"rolling_std_7",
"rolling_mean_14",
"rolling_std_14",
"rolling_mean_30",
"rolling_std_30",
"rolling_mean_60",
"rolling_std_60",
"rolling_mean_180",
"rolling_std_180",
"rolling_mean_tmp_1_7",
"rolling_mean_tmp_1_14",
"rolling_mean_tmp_1_30",
"rolling_mean_tmp_1_60",
"rolling_mean_tmp_7_7",
"rolling_mean_tmp_7_14",
"rolling_mean_tmp_7_30",
"rolling_mean_tmp_7_60",
"rolling_mean_tmp_14_7",
"rolling_mean_tmp_14_14",
"rolling_mean_tmp_14_30",
"rolling_mean_tmp_14_60",
# "enc_state_id_mean",
# "enc_state_id_std",
# "enc_store_id_mean",
# "enc_store_id_std",
"enc_cat_id_mean",
"enc_cat_id_std",
"enc_dept_id_mean",
"enc_dept_id_std",
"enc_state_id_cat_id_mean",
"enc_state_id_cat_id_std",
"enc_state_id_dept_id_mean",
"enc_state_id_dept_id_std",
"enc_store_id_cat_id_mean",
"enc_store_id_cat_id_std",
"enc_store_id_dept_id_mean",
"enc_store_id_dept_id_std",
"enc_item_id_mean",
"enc_item_id_std",
"enc_item_id_state_id_mean",
"enc_item_id_state_id_std",
"enc_item_id_store_id_mean",
"enc_item_id_store_id_std",
]
label_column = ["sales"]


lgb_params_list = [
[0.015, 224, 66],
[0.01, 224, 50],
[0.01, 300, 80],
[0.015, 128, 50],
[0.015, 300, 50],
[0.01, 300, 66],
[0.015, 300, 80],
[0.15, 224, 80],
[0.005, 300, 50],
[0.015, 224, 50],
]


store_list = ["CA_1", "CA_2", "CA_3", "CA_4", "TX_1", "TX_2", "TX_3", "WI_1", "WI_2", "WI_3"]
dataset_info = {
"name": "M5",
"range of date": "2011.01.29-2016.06.19",
"description": "Walmart store, involves the unit sales of various products sold in the USA, organized in the form of grouped time series. More specifically, the dataset involves the unit sales of 3049 products, classified in 3 product categories (Hobbies, Foods, and Household).",
"location": [
"California, United States",
"California, United States",
"California, United States",
"California, United States",
"Texas, United States",
"Texas, United States",
"Texas, United States",
"Wisconsin, United States",
"Wisconsin, United States",
"Wisconsin, United States",
],
}

+ 0
- 338
examples/dataset_m5_workflow/m5/generate_data.py View File

@@ -1,338 +0,0 @@
import numpy as np
import pandas as pd
from math import ceil
from tqdm import tqdm
from copy import deepcopy as dco
import os, sys, gc, time, warnings, pickle, psutil, random
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import MinMaxScaler


from .utils import *
from .config import raw_data_dir, processed_data_dir, TARGET

warnings.filterwarnings("ignore")


# ==================== preprocessing ====================
def melt_raw_data(train_df):
if os.path.exists(os.path.join(processed_data_dir, "melt_raw_data.pkl")):
return pd.read_pickle(os.path.join(processed_data_dir, "melt_raw_data.pkl"))

index_columns = ["id", "item_id", "dept_id", "cat_id", "store_id", "state_id"]
grid_df = pd.melt(train_df, id_vars=index_columns, var_name="d", value_name=TARGET)

for col in index_columns:
grid_df[col] = grid_df[col].astype("category")

grid_df.to_pickle(os.path.join(processed_data_dir, "melt_raw_data.pkl"))
return grid_df


def add_release_week(grid_df, prices_df, calendar_df):
if os.path.exists(os.path.join(processed_data_dir, "add_release_week.pkl")):
return pd.read_pickle(os.path.join(processed_data_dir, "add_release_week.pkl"))

release_df = prices_df.groupby(["store_id", "item_id"])["wm_yr_wk"].agg(["min"]).reset_index()
release_df.columns = ["store_id", "item_id", "release"]
grid_df = merge_by_concat(grid_df, release_df, ["store_id", "item_id"])
grid_df = merge_by_concat(grid_df, calendar_df[["wm_yr_wk", "d"]], ["d"])

# cutoff meaningless rows
grid_df = grid_df[grid_df["wm_yr_wk"] >= grid_df["release"]]
grid_df = grid_df.reset_index(drop=True)

# scale the release
grid_df["release"] = grid_df["release"] - grid_df["release"].min()
grid_df["release"] = grid_df["release"].astype(np.int16)

grid_df.to_pickle(os.path.join(processed_data_dir, "add_release_week.pkl"))
return grid_df


def add_prices(grid_df, prices_df, calendar_df):
if os.path.exists(os.path.join(processed_data_dir, "add_prices.pkl")):
return pd.read_pickle(os.path.join(processed_data_dir, "add_prices.pkl"))

prices_df["price_max"] = prices_df.groupby(["store_id", "item_id"])["sell_price"].transform("max")
prices_df["price_min"] = prices_df.groupby(["store_id", "item_id"])["sell_price"].transform("min")
prices_df["price_std"] = prices_df.groupby(["store_id", "item_id"])["sell_price"].transform("std")
prices_df["price_mean"] = prices_df.groupby(["store_id", "item_id"])["sell_price"].transform("mean")
prices_df["price_norm"] = prices_df["sell_price"] / prices_df["price_max"]

prices_df["price_nunique"] = prices_df.groupby(["store_id", "item_id"])["sell_price"].transform("nunique")
prices_df["item_nunique"] = prices_df.groupby(["store_id", "sell_price"])["item_id"].transform("nunique")

calendar_prices = calendar_df[["wm_yr_wk", "month", "year"]]
calendar_prices = calendar_prices.drop_duplicates(subset=["wm_yr_wk"])
prices_df = prices_df.merge(calendar_prices[["wm_yr_wk", "month", "year"]], on=["wm_yr_wk"], how="left")

prices_df["price_momentum"] = prices_df["sell_price"] / prices_df.groupby(["store_id", "item_id"])[
"sell_price"
].transform(lambda x: x.shift(1))
prices_df["price_momentum_m"] = prices_df["sell_price"] / prices_df.groupby(["store_id", "item_id", "month"])[
"sell_price"
].transform("mean")
prices_df["price_momentum_y"] = prices_df["sell_price"] / prices_df.groupby(["store_id", "item_id", "year"])[
"sell_price"
].transform("mean")

grid_df = reduce_mem_usage(grid_df)
prices_df = reduce_mem_usage(prices_df)

original_columns = list(grid_df)
grid_df = grid_df.merge(prices_df, on=["store_id", "item_id", "wm_yr_wk"], how="left")
grid_df = reduce_mem_usage(grid_df)

grid_df.to_pickle(os.path.join(processed_data_dir, "add_prices.pkl"))
return grid_df


def add_date(grid_df, calendar_df):
if os.path.exists(os.path.join(processed_data_dir, "add_date.pkl")):
return pd.read_pickle(os.path.join(processed_data_dir, "add_date.pkl"))

# merge calendar partly
icols = [
"date",
"d",
"event_name_1",
"event_type_1",
"event_name_2",
"event_type_2",
"snap_CA",
"snap_TX",
"snap_WI",
]
grid_df = grid_df.merge(calendar_df[icols], on=["d"], how="left")

# convert to category
icols = [
"event_name_1",
"event_type_1",
"event_name_2",
"event_type_2",
"snap_CA",
"snap_TX",
"snap_WI",
]
for col in icols:
grid_df[col] = grid_df[col].astype("category")

# make some features from date
grid_df["date"] = pd.to_datetime(grid_df["date"])
grid_df["tm_d"] = grid_df["date"].dt.day.astype(np.int8)
grid_df["tm_w"] = grid_df["date"].dt.week.astype(np.int8)
grid_df["tm_m"] = grid_df["date"].dt.month.astype(np.int8)
grid_df["tm_y"] = grid_df["date"].dt.year
grid_df["tm_y"] = (grid_df["tm_y"] - grid_df["tm_y"].min()).astype(np.int8)
grid_df["tm_wm"] = grid_df["tm_d"].apply(lambda x: ceil(x / 7)).astype(np.int8)

grid_df["tm_dw"] = grid_df["date"].dt.dayofweek.astype(np.int8)
grid_df["tm_w_end"] = (grid_df["tm_dw"] >= 5).astype(np.int8)

# clear columns
grid_df["d"] = grid_df["d"].apply(lambda x: x[2:]).astype(np.int16)
grid_df = grid_df.drop("wm_yr_wk", 1)

grid_df.to_pickle(os.path.join(processed_data_dir, "add_date.pkl"))
return grid_df


def add_lags_rollings(grid_df):
if os.path.exists(os.path.join(processed_data_dir, "add_lags_rollings.pkl")):
return pd.read_pickle(os.path.join(processed_data_dir, "add_lags_rollings.pkl"))

# add lags
SHIFT_DAY = 28
LAG_DAYS = [col for col in range(SHIFT_DAY, SHIFT_DAY + 15)]

grid_df = grid_df.assign(
**{
"{}_lag_{}".format(col, l): grid_df.groupby(["id"])[col].transform(lambda x: x.shift(l))
for l in LAG_DAYS
for col in [TARGET]
}
)

for col in list(grid_df):
if "lag" in col:
grid_df[col] = grid_df[col].astype(np.float16)

# add rollings
for i in [7, 14, 30, 60, 180]:
grid_df["rolling_mean_" + str(i)] = (
grid_df.groupby(["id"])[TARGET].transform(lambda x: x.shift(SHIFT_DAY).rolling(i).mean()).astype(np.float16)
)
grid_df["rolling_std_" + str(i)] = (
grid_df.groupby(["id"])[TARGET].transform(lambda x: x.shift(SHIFT_DAY).rolling(i).std()).astype(np.float16)
)

# sliding window
for d_shift in [1, 7, 14]:
for d_window in [7, 14, 30, 60]:
col_name = "rolling_mean_tmp_" + str(d_shift) + "_" + str(d_window)
grid_df[col_name] = (
grid_df.groupby(["id"])[TARGET]
.transform(lambda x: x.shift(SHIFT_DAY + d_shift).rolling(d_window).mean())
.astype(np.float16)
)

grid_df.to_pickle(os.path.join(processed_data_dir, "add_lags_rollings.pkl"))
return grid_df


def add_mean_enc(grid_df):
if os.path.exists(os.path.join(processed_data_dir, "add_mean_enc.pkl")):
return pd.read_pickle(os.path.join(processed_data_dir, "add_mean_enc.pkl"))

sales_df = dco(grid_df["sales"])
grid_df["sales"][grid_df["d"] > (1941 - 28)] = np.nan

icols = [
["state_id"],
["store_id"],
["cat_id"],
["dept_id"],
["state_id", "cat_id"],
["state_id", "dept_id"],
["store_id", "cat_id"],
["store_id", "dept_id"],
["item_id"],
["item_id", "state_id"],
["item_id", "store_id"],
]

for col in icols:
col_name = "_" + "_".join(col) + "_"
grid_df["enc" + col_name + "mean"] = grid_df.groupby(col)["sales"].transform("mean").astype(np.float16)
grid_df["enc" + col_name + "std"] = grid_df.groupby(col)["sales"].transform("std").astype(np.float16)

grid_df["sales"] = sales_df

grid_df.to_pickle(os.path.join(processed_data_dir, "add_mean_enc.pkl"))
return grid_df


def add_snap(grid_df):
if os.path.exists(os.path.join(processed_data_dir, "all_data_df.pkl")):
return pd.read_pickle(os.path.join(processed_data_dir, "all_data_df.pkl"))

mask_CA = grid_df["state_id"] == "CA"
mask_WI = grid_df["state_id"] == "WI"
mask_TX = grid_df["state_id"] == "TX"

grid_df["snap"] = grid_df["snap_CA"]
grid_df.loc[mask_WI, "snap"] = grid_df["snap_WI"]
grid_df.loc[mask_TX, "snap"] = grid_df["snap_TX"]

grid_df.to_pickle(os.path.join(processed_data_dir, "all_data_df.pkl"))
return grid_df


def preprocessing_m5():
train_df = pd.read_csv(os.path.join(raw_data_dir, "sales_train_evaluation.csv"))
prices_df = pd.read_csv(os.path.join(raw_data_dir, "sell_prices.csv"))
calendar_df = pd.read_csv(os.path.join(raw_data_dir, "calendar.csv"))

grid_df = melt_raw_data(train_df)
print(f"df: ({grid_df.shape[0]}, {grid_df.shape[1]}) Melting raw data down!")

grid_df = add_release_week(grid_df, prices_df, calendar_df)
print(f"df: ({grid_df.shape[0]}, {grid_df.shape[1]}) Adding release week down!")

grid_df = add_prices(grid_df, prices_df, calendar_df)
print(f"df: ({grid_df.shape[0]}, {grid_df.shape[1]}) Adding prices down!")

grid_df = add_date(grid_df, calendar_df)
print(f"df: ({grid_df.shape[0]}, {grid_df.shape[1]}) Adding date down!")

grid_df = add_lags_rollings(grid_df)
print(f"df: ({grid_df.shape[0]}, {grid_df.shape[1]}) Adding lags and rollings down!")

grid_df = add_mean_enc(grid_df)
print(f"df: ({grid_df.shape[0]}, {grid_df.shape[1]}) Adding mean encoding down!")

grid_df = pd.read_pickle(os.path.join(processed_data_dir, "add_mean_enc.pkl"))

grid_df = add_snap(grid_df)
print("Save the data down!")


# ==================== split dataset ====================
def label_encode(df, columns):
le = LabelEncoder()
data_list = []

for column in columns:
data_list += df[column].drop_duplicates().values.tolist()
le.fit(data_list)

for column in columns:
df[column] = le.transform(df[column].values.tolist())

return df


def reorganize_data(grid_df):
grid_df["snap"] = grid_df["snap"].astype("int8")
columns_list = [
["item_id"],
["dept_id"],
["cat_id"],
["event_name_1", "event_name_2"],
["event_type_1", "event_type_2"],
]

for columns in columns_list:
grid_df[columns] = label_encode(grid_df[columns], columns)

return reduce_mem_usage(grid_df)


def split_data(df, store, fill_flag=False):
for cat in category_list:
df[cat] = df[cat].astype("category")

if fill_flag:
df = reduce_mem_usage(df, float16_flag=False)
cols = df.isnull().any()
idx = list(cols[cols.values].index)

df[idx] = df.groupby("item_id", sort=False)[idx].apply(lambda x: x.ffill().bfill())
df[idx] = df[idx].fillna(df[idx].mean())

mms = MinMaxScaler()
df[features_columns] = mms.fit_transform(df[features_columns])

df = reduce_mem_usage(df)

train_df = df[df["d"] <= END_TRAIN]
val_df = df[df["d"] > END_TRAIN]

train_df = train_df[features_columns + label_column]
val_df = val_df[features_columns + label_column]
print(train_df.shape, val_df.shape)

suffix = f"_fill" if fill_flag else ""
train_df.to_pickle(os.path.join(processed_data_dir, f"train_{store}{suffix}.pkl"))
val_df.to_pickle(os.path.join(processed_data_dir, f"val_{store}{suffix}.pkl"))


def split_m5():
grid_df = pd.read_pickle(os.path.join(processed_data_dir, "all_data_df.pkl"))

if os.path.exists(os.path.join(processed_data_dir, "label_encode.pkl")):
grid_df = pd.read_pickle(os.path.join(processed_data_dir, "label_encode.pkl"))
else:
grid_df = reorganize_data(grid_df)
grid_df.to_pickle(os.path.join(processed_data_dir, "label_encode.pkl"))

for store in store_list:
# split_data(grid_df[grid_df["store_id"] == store], store)
split_data(grid_df[grid_df["store_id"] == store], store, True)


def regenerate_data():
preprocessing_m5()
split_m5()

+ 0
- 452
examples/dataset_m5_workflow/m5/train.py View File

@@ -1,452 +0,0 @@
import gc
import joblib
import random
import numpy as np
import pandas as pd
from tqdm import tqdm
import os, warnings
import lightgbm as lgb
from sklearn.svm import SVR
from sklearn.linear_model import Ridge
from sklearn.kernel_ridge import KernelRidge
from sklearn.metrics import mean_squared_error
from sklearn.metrics.pairwise import rbf_kernel


from .utils import *
from .config import model_dir, grid_dir, store_list, lgb_params_list

warnings.filterwarnings("ignore")


def train_lgb_model(train_x, train_y, val_x, val_y, store, lr, nl, md, best, save=True, n_estimators=0, train_flag=0):
lgb_params = {
"boosting_type": "gbdt",
"objective": "rmse",
"metric": "rmse",
"learning_rate": lr,
"num_leaves": nl,
"max_depth": md,
"n_estimators": 100000,
"boost_from_average": False,
"verbose": -1,
}

if train_flag:
idx = int(len(train_y) * 0.1)
train_data = lgb.Dataset(train_x[:-idx], label=train_y[:-idx])
val_data = lgb.Dataset(train_x[-idx:], label=train_y[-idx:])
else:
train_data = lgb.Dataset(train_x, label=train_y)
val_data = lgb.Dataset(val_x, label=val_y)

if n_estimators:
lgb_params["n_estimators"] = n_estimators
gbm = lgb.train(lgb_params, train_data, verbose_eval=100)
else:
gbm = lgb.train(lgb_params, train_data, valid_sets=[val_data], verbose_eval=100, early_stopping_rounds=1000)

test_y = gbm.predict(val_x, num_iteration=gbm.best_iteration)
res = mean_squared_error(val_y, test_y, squared=False)

if res < best:
best = res
if save:
gbm.save_model(os.path.join(model_dir, f"lgb_{store}.out"))

return best


def train_ridge_model(train_x, train_y, val_x, val_y, store, a, best, save=True):
model = Ridge(alpha=a)
model.fit(train_x, train_y)

test_y = model.predict(val_x)
res = mean_squared_error(val_y, test_y, squared=False)

if res < best:
best = res
if save:
joblib.dump(model, os.path.join(model_dir, f"ridge_{store}.out"))

return best


def train_svm_model(
train_x, train_y, val_x, val_y, store, C, epsilon, best, save=True, gamma=0.1, adaptation_model=[], K1=None, K2=None
):
if K1 is None:
model = SVR(C=C, epsilon=epsilon, max_iter=30000, cache_size=10240, verbose=True, gamma=gamma)
else:
model = AuxiliarySVR(
C=C,
epsilon=epsilon,
gamma=gamma,
adaptation_model=adaptation_model,
max_iter=30000,
cache_size=10240,
verbose=True,
K1=K1,
K2=K2,
)

model.fit(train_x, train_y)
test_y = model.predict(val_x)
res = mean_squared_error(val_y, test_y, squared=False)

if res < best:
best = res
if save:
joblib.dump(model, os.path.join(model_dir, f"svm_{store}.out"))

return best


def train_krr_model(train_x, train_y, val_x, val_y, store, a, best, save=True, gamma=0.1, K1=None, K2=None):
if K1 is None:
model = KernelRidge(kernel="rbf", alpha=a, gamma=gamma)
model.fit(train_x, train_y)
test_y = model.predict(val_x)
res = mean_squared_error(val_y, test_y, squared=False)
else:
len1, len2 = len(train_y), len(val_y)
model = KernelRidge(kernel="precomputed", alpha=a)
model.fit(K1[-len1:, -len1:], train_y)
test_y = model.predict(K2[-len2:, -len1:])
res = mean_squared_error(val_y, test_y, squared=False)

if res < best:
best = res
if save:
joblib.dump(model, os.path.join(model_dir, f"krr_{store}.out"))

return best


def grid_search(store_id, algo, search_lgb_flag=False):
store = store_list[store_id]

if algo == "lgb":
train_x, train_y, val_x, val_y = acquire_data(store, True)
learning_rate = [0.005, 0.01, 0.015]
num_leaves = [128, 224, 300]
max_depth = [50, 66, 80]
best = 10000000

if search_lgb_flag:
for lr in learning_rate:
for nl in num_leaves:
for md in max_depth:
best = train_lgb_model(train_x, train_y, val_x, val_y, store, lr, nl, md, best)
print(f"store: {store}, lr: {lr}, nl: {nl}, md: {md}, best: {best}")
else:
lr, nl, md = lgb_params_list[store_id]
best = train_lgb_model(train_x, train_y, val_x, val_y, store, lr, nl, md, best)
print(f"store: {store}, lr: {lr}, nl: {nl}, md: {md}, best: {best}")
elif algo == "ridge":
train_x, train_y, val_x, val_y = acquire_data(store, True)
alpha = [0.01, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10, 20, 30]
best = 10000000

for a in alpha:
best = train_ridge_model(train_x, train_y, val_x, val_y, store, a, best)
print(f"store: {store}, alpha: {a}, best: {best}")


def grid_training_sample(algo, user_list=list(range(10))):
for i in range(len(user_list)):
store_id = user_list[i]
store = store_list[store_id]
org_train_x, org_train_y, val_x, val_y = acquire_data(store, True)
res = []

proportion_list = [
100,
300,
500,
700,
900,
1000,
3000,
5000,
7000,
9000,
10000,
30000,
50000,
70000,
90000,
100000,
300000,
500000,
700000,
900000,
1000000,
3000000,
5000000,
]

for proportion in proportion_list:
"""
random
org_idx_list = list(range(len(org_train_y)))
idx_list = random.sample(org_idx_list, min(proportion, len(org_train_y)))
train_x = org_train_x.iloc[idx_list]
train_y = org_train_y.iloc[idx_list]
"""
train_x = org_train_x[-proportion:]
train_y = org_train_y[-proportion:]
best = 10000000

if algo == "lgb":
lr, nl, md = lgb_params_list[store_id]
best = train_lgb_model(
train_x, train_y, val_x, val_y, store, lr, nl, md, best, save=False, n_estimators=3000, train_flag=0
)
print(f"store: {store}, lr: {lr}, nl: {nl}, md: {md}, best: {best}")

elif algo == "ridge":
alpha = [0.01, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10, 20, 30]
for a in alpha:
best = train_ridge_model(train_x, train_y, val_x, val_y, store, a, best, save=False)
print(f"store: {store}, alpha: {a}, best: {best}")

elif algo == "svm":
C = [1, 10, 100]
epsilon = 0.001
for c in C:
best = train_svm_model(train_x, train_y, val_x, val_y, store, c, epsilon, best, save=False)
print(f"store: {store}, C: {c}, epsilon: {epsilon}, best: {best}")

res.append([proportion, best])
np.savetxt(os.path.join(grid_dir, f"grid_sample_{algo}_{store}.out"), np.array(res))

if proportion > len(org_train_y):
break


def retrain_models(algo):
for store_id in range(10):
grid_search(store_id, algo)


def train_adaptation_grid(
algo, max_sample, test_sample, user_list=list(range(10)), adaptation_model=[], residual=False
):
"""
adaptation_model = [
[("lgb", 1), ("ridge", 2)],
[("lgb", 1), ("ridge", 2)]
]
"""

proportion_list = [
100,
300,
500,
700,
900,
1000,
3000,
5000,
7000,
9000,
10000,
30000,
50000,
70000,
90000,
100000,
300000,
500000,
700000,
900000,
1000000,
3000000,
5000000,
]
sample_idx = proportion_list.index(max_sample) + 1

for i in range(len(user_list)):
store_id = user_list[i]
store = store_list[store_id]
org_train_x, org_train_y, val_x, val_y = acquire_data(store, True)
val_x = val_x[-test_sample:]
val_y = val_y[-test_sample:]

if algo == "lgb" or algo == "ridge":
res = []

if adaptation_model != []:
if residual:
aux_algo, model_idx = adaptation_model[i][0]
org_train_y -= model_predict(aux_algo, model_idx, org_train_x)
val_y -= model_predict(aux_algo, model_idx, val_x)

else:
train_y_list, val_y_list = [], []

for aux_algo, model_idx in adaptation_model[i]:
train_y_list.append(model_predict(aux_algo, model_idx, org_train_x))
val_y_list.append(model_predict(aux_algo, model_idx, val_x))

for j in range(len(train_y_list)):
org_train_x[f"model_values_{j}"] = train_y_list[j]
val_x[f"model_values_{j}"] = val_y_list[j]

for proportion in proportion_list[:sample_idx]:
"""
random
org_idx_list = list(range(len(org_train_y)))
idx_list = random.sample(org_idx_list, min(proportion, len(org_train_y)))
train_x = org_train_x.iloc[idx_list]
train_y = org_train_y.iloc[idx_list]
"""
train_x = org_train_x[-proportion:]
train_y = org_train_y[-proportion:]
best = 10000000

if algo == "lgb":
if max_sample < 50000:
learning_rate = [0.005, 0.01, 0.015]
num_leaves = [128, 224, 300]
max_depth = [50, 66, 80]

for lr in learning_rate:
for nl in num_leaves:
for md in max_depth:
best = train_lgb_model(
train_x, train_y, val_x, val_y, store, lr, nl, md, best, save=False
)
print(f"store: {store}, lr: {lr}, nl: {nl}, md: {md}, best: {best}")
else:
lr, nl, md = lgb_params_list[store_id]
best = train_lgb_model(
train_x,
train_y,
val_x,
val_y,
store,
lr,
nl,
md,
best,
save=False,
n_estimators=3000,
train_flag=0,
)
print(f"store: {store}, lr: {lr}, nl: {nl}, md: {md}, best: {best}")

elif algo == "ridge":
alpha = [0.01, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10, 20, 30]
for a in alpha:
best = train_ridge_model(train_x, train_y, val_x, val_y, store, a, best, save=False)
print(f"store: {store}, alpha: {a}, best: {best}")

res.append([proportion, best])
text = str(adaptation_model[i]) if adaptation_model != [] else "null"
text += "_residual_" if residual else ""
np.savetxt(os.path.join(grid_dir, f"{algo}_using_{text}_{store}.out"), np.array(res))

if proportion > len(org_train_y):
break

elif algo == "svm" or algo == "krr":
res = [[proportion, 10000] for proportion in proportion_list[:sample_idx]]
org_train_x = org_train_x.to_numpy()
org_train_y = org_train_y.to_numpy()
val_x = val_x.to_numpy()
val_y = val_y.to_numpy()

y1_list, y2_list = [], []
gamma_list = [0.01, 0.1, 0.5, 1]

if residual:
aux_algo, model_idx = adaptation_model[i][0]
org_train_y = org_train_y.astype(np.float64)
val_y = val_y.astype(np.float64)
org_train_y -= model_predict(aux_algo, model_idx, org_train_x)
val_y -= model_predict(aux_algo, model_idx, val_x)

elif adaptation_model != []:
for aux_algo, idx in adaptation_model[i]:
y1_list.append(model_predict(aux_algo, idx, org_train_x[-max_sample:]).reshape(-1, 1))
y2_list.append(model_predict(aux_algo, idx, val_x).reshape(-1, 1))

for gamma in gamma_list:
K1 = np.zeros((max_sample, max_sample))
K2 = np.zeros((len(val_x), max_sample))

if (not residual) and adaptation_model != []:
for j in range(len(adaptation_model[i])):
aux_algo, idx = adaptation_model[i][j]
y1 = y1_list[j]
y2 = y2_list[j]
K1 += np.dot(y1, y1.T)
K2 += np.dot(y2, y1.T)

K1 += rbf_kernel(org_train_x[-max_sample:], org_train_x[-max_sample:], gamma=gamma)
K2 += rbf_kernel(val_x, org_train_x[-max_sample:], gamma=gamma)

for idx in range(len(proportion_list[:sample_idx])):
proportion = proportion_list[idx]
"""
random
org_idx_list = list(range(len(org_train_y)))
idx_list = random.sample(org_idx_list, min(proportion, len(org_train_y)))
train_x = org_train_x.iloc[idx_list]
train_y = org_train_y.iloc[idx_list]
"""
train_x = org_train_x[-proportion:]
train_y = org_train_y[-proportion:]
best = 10000000

if algo == "svm":
C = [1, 10, 50, 100, 200]
epsilon = 0.001

for c in C:
adapt_m = [] if adaptation_model == [] else adaptation_model[i]
best = train_svm_model(
train_x,
train_y,
val_x,
val_y,
store,
c,
epsilon,
best,
save=False,
gamma=gamma,
adaptation_model=adapt_m,
K1=K1,
K2=K2,
)
print(f"store: {store}, gamma: {gamma}, C: {c}, epsilon: {epsilon}, best: {best}")

elif algo == "krr":
alpha = [0.01, 0.1, 0.5, 1.0, 5.0, 10]

for a in alpha:
best = train_krr_model(
train_x, train_y, val_x, val_y, store, a, best, save=False, gamma=gamma, K1=K1, K2=K2
)
print(f"store: {store}, a: {a}, gamma: {gamma}, best: {best}")

if best < res[idx][1]:
res[idx][1] = best
text = str(adaptation_model[i]) if adaptation_model != [] else "null"
text += "_residual" if residual else ""
np.savetxt(os.path.join(grid_dir, f"{algo}_using_{text}_{store}.out"), np.array(res))

if proportion > len(org_train_y):
break

del train_x, train_y
gc.collect()

del K1, K2
gc.collect()

del org_train_x, org_train_y
gc.collect()

+ 0
- 177
examples/dataset_m5_workflow/m5/utils.py View File

@@ -1,177 +0,0 @@
from math import gamma
from tkinter import Y
import joblib
from tqdm import tqdm
import numpy as np
import pandas as pd
import lightgbm as lgb
from sklearn.svm import SVR
from sklearn.metrics import mean_squared_error
from sklearn.metrics.pairwise import rbf_kernel
import os, sys, gc, time, warnings, pickle, psutil, random
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import make_axes_locatable


from .config import *


class AuxiliarySVR:
def __init__(
self, C, epsilon, gamma, adaptation_model=[], max_iter=30000, cache_size=10240, verbose=False, K1=None, K2=None
):
self.gamma = gamma
self.adaptation_model = adaptation_model
self.model = SVR(
C=C,
epsilon=epsilon,
kernel=self.auxiliary_rbf_kernel,
max_iter=max_iter,
cache_size=cache_size,
verbose=verbose,
)
self.K1 = K1
self.K2 = K2

def auxiliary_rbf_kernel(self, X1, X2):
if self.K1 is not None:
if X1.shape[0] == X2.shape[0]:
return self.K1[-X1.shape[0] :, -X2.shape[0] :]
else:
return self.K2[-X1.shape[0] :, -X2.shape[0] :]
else:
K = np.zeros((len(X1), len(X2)))

for algo, idx in self.adaptation_model:
Y1 = model_predict(algo, idx, X1).reshape(-1, 1)
Y2 = model_predict(algo, idx, X2).reshape(-1, 1)
K += Y1 @ Y2.T

K += rbf_kernel(X1, X2, self.gamma)
return K

def fit(self, X, Y):
self.gamma = 1 / X.shape[1]
self.model.fit(X, Y)

def predict(self, X):
return self.model.predict(X)


def measure_aux_algo(idx, test_sample, model):
"""
model = ("lgb", 1)
"""
store = store_list[idx]
org_train_x, org_train_y, val_x, val_y = acquire_data(store, True)
pred_y = model_predict(model[0], model[1], val_x[-test_sample:])
return score(pred_y, val_y[-test_sample:])


# Simple "Memory profilers" to see memory usage
def get_memory_usage():
return np.round(psutil.Process(os.getpid()).memory_info()[0] / 2.0**30, 2)


def sizeof_fmt(num, suffix="B"):
for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]:
if abs(num) < 1024.0:
return "%3.1f%s%s" % (num, unit, suffix)
num /= 1024.0
return "%.1f%s%s" % (num, "Yi", suffix)


# Memory Reducer
def reduce_mem_usage(df, float16_flag=True, verbose=True):
numerics = ["int16", "int32", "int64", "float16", "float32", "float64"]
start_mem = df.memory_usage().sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == "int":
if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
df[col] = df[col].astype(np.int8)
elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
df[col] = df[col].astype(np.int16)
elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
df[col] = df[col].astype(np.int32)
elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
df[col] = df[col].astype(np.int64)
else:
if float16_flag and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
df[col] = df[col].astype(np.float16)
elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
df[col] = df[col].astype(np.float32)
else:
df[col] = df[col].astype(np.float64)
end_mem = df.memory_usage().sum() / 1024**2
if verbose:
print(
"Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)".format(
end_mem, 100 * (start_mem - end_mem) / start_mem
)
)
return df


# Merging by concat to not lose dtypes
def merge_by_concat(df1, df2, merge_on):
merged_gf = df1[merge_on]
merged_gf = merged_gf.merge(df2, on=merge_on, how="left")
new_columns = [col for col in list(merged_gf) if col not in merge_on]
df1 = pd.concat([df1, merged_gf[new_columns]], axis=1)
return df1


def model_predict(algo, idx, test_x):
store = store_list[idx]

if algo == "lgb":
model = lgb.Booster(model_file=os.path.join(model_dir, f"lgb_{store}.out"))
return model.predict(test_x, num_iteration=model.best_iteration)
elif algo == "ridge":
model = joblib.load(os.path.join(model_dir, f"ridge_{store}.out"))
return model.predict(test_x)
elif algo == "svm":
model = joblib.load(os.path.join(model_dir, f"svm_{store}.out"))
return model.predict(test_x)


def get_weights(algo):
weights = []

if algo == "lgb":
for store in store_list:
model = lgb.Booster(model_file=os.path.join(model_dir, f"lgb_{store}.out"))
weights.append(model.feature_importance())
else:
for store in store_list:
model = joblib.load(os.path.join(model_dir, f"ridge_{store}.out"))
weights.append(model.coef_)

return np.array(weights)


def score(real_y, pred_y, sample_weight, multioutput):
return mean_squared_error(real_y, pred_y, sample_weight=sample_weight, multioutput=multioutput, squared=False)


def acquire_data(store, fill_flag=False):
TARGET = "sales"
suffix = f"_fill" if fill_flag else ""
train = pd.read_pickle(os.path.join(processed_data_dir, f"train_{store}{suffix}.pkl"))
val = pd.read_pickle(os.path.join(processed_data_dir, f"val_{store}{suffix}.pkl"))

train_y = train[TARGET]
train_x = train.drop(columns=TARGET, axis=1)
val_y = val[TARGET]
val_x = val.drop(columns=TARGET, axis=1)

train_x = train_x.to_numpy()
train_y = train_y.to_numpy()
val_x = val_x.to_numpy()
val_y = val_y.to_numpy()

return train_x, train_y, val_x, val_y

+ 0
- 211
examples/dataset_m5_workflow/main.py View File

@@ -1,211 +0,0 @@
import os
import fire
import time
import zipfile
import numpy as np
from tqdm import tqdm
from shutil import copyfile, rmtree

import learnware
from learnware.market import instantiate_learnware_market, BaseUserInfo
from learnware.reuse import JobSelectorReuser, AveragingReuser
from learnware.specification import generate_rkme_table_spec
from m5 import DataLoader
from learnware.logger import get_module_logger

logger = get_module_logger("m5_test", level="INFO")


output_description = {
"Dimension": 1,
"Description": {},
}

input_description = {
"Dimension": 82,
"Description": {},
}

semantic_specs = [
{
"Data": {"Values": ["Table"], "Type": "Class"},
"Task": {"Values": ["Regression"], "Type": "Class"},
"Library": {"Values": ["Scikit-learn"], "Type": "Class"},
"Scenario": {"Values": ["Business"], "Type": "Tag"},
"Description": {"Values": "", "Type": "String"},
"Name": {"Values": "learnware_1", "Type": "String"},
"Input": input_description,
"Output": output_description,
"License": {"Values": ["MIT"], "Type": "Class"},
}
]

user_semantic = {
"Data": {"Values": ["Table"], "Type": "Class"},
"Task": {"Values": ["Regression"], "Type": "Class"},
"Library": {"Values": ["Scikit-learn"], "Type": "Class"},
"Scenario": {"Values": ["Business"], "Type": "Tag"},
"Description": {"Values": "", "Type": "String"},
"Name": {"Values": "", "Type": "String"},
"Input": input_description,
"Output": output_description,
"License": {"Values": ["MIT"], "Type": "Class"},
}


class M5DatasetWorkflow:
def _init_m5_dataset(self):
m5 = DataLoader()
m5.regenerate_data()

algo_list = ["ridge", "lgb"]
for algo in algo_list:
m5.set_algo(algo)
m5.retrain_models()

def _init_learnware_market(self):
"""initialize learnware market"""
# database_ops.clear_learnware_table()
learnware.init()

easy_market = instantiate_learnware_market(name="easy", rebuild=True)
print("Total Item:", len(easy_market))

zip_path_list = []
curr_root = os.path.dirname(os.path.abspath(__file__))
curr_root = os.path.join(curr_root, "learnware_pool")
for zip_path in os.listdir(curr_root):
zip_path_list.append(os.path.join(curr_root, zip_path))

for idx, zip_path in enumerate(zip_path_list):
semantic_spec = semantic_specs[0]
semantic_spec["Name"]["Values"] = "learnware_%d" % (idx)
semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx)
easy_market.add_learnware(zip_path, semantic_spec)

print("Total Item:", len(easy_market))

def prepare_learnware(self, regenerate_flag=False):
if regenerate_flag:
self._init_m5_dataset()

m5 = DataLoader()
idx_list = m5.get_idx_list()
algo_list = ["lgb"] # algo_list = ["ridge", "lgb"]

curr_root = os.path.dirname(os.path.abspath(__file__))
curr_root = os.path.join(curr_root, "learnware_pool")
os.makedirs(curr_root, exist_ok=True)

for idx in tqdm(idx_list):
train_x, train_y, test_x, test_y = m5.get_idx_data(idx)
st = time.time()
spec = generate_rkme_table_spec(X=train_x, gamma=0.1, cuda_idx=0)
ed = time.time()
logger.info("Stat spec generated in %.3f s" % (ed - st))

for algo in algo_list:
m5.set_algo(algo)
dir_path = os.path.join(curr_root, f"{algo}_{idx}")
os.makedirs(dir_path, exist_ok=True)

spec_path = os.path.join(dir_path, "rkme.json")
spec.save(spec_path)

model_path = m5.get_model_path(idx)
model_file = os.path.join(dir_path, "model.out")
copyfile(model_path, model_file)

init_file = os.path.join(dir_path, "__init__.py")
copyfile("example_init.py", init_file)

yaml_file = os.path.join(dir_path, "learnware.yaml")
copyfile("example.yaml", yaml_file)

zip_file = dir_path + ".zip"
with zipfile.ZipFile(zip_file, "w") as zip_obj:
for foldername, subfolders, filenames in os.walk(dir_path):
for filename in filenames:
file_path = os.path.join(foldername, filename)
zip_info = zipfile.ZipInfo(filename)
zip_info.compress_type = zipfile.ZIP_STORED
with open(file_path, "rb") as file:
zip_obj.writestr(zip_info, file.read())

rmtree(dir_path)

def test(self, regenerate_flag=False):
self.prepare_learnware(regenerate_flag)
self._init_learnware_market()

easy_market = instantiate_learnware_market(name="easy")
print("Total Item:", len(easy_market))

m5 = DataLoader()
idx_list = m5.get_idx_list()
os.makedirs("./user_spec", exist_ok=True)
single_score_list = []
random_score_list = []
job_selector_score_list = []
ensemble_score_list = []
improve_list = []

for idx in idx_list:
train_x, train_y, test_x, test_y = m5.get_idx_data(idx)
user_spec = generate_rkme_table_spec(X=test_x, gamma=0.1, cuda_idx=0)
user_spec_path = f"./user_spec/user_{idx}.json"
user_spec.save(user_spec_path)

user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": user_spec})
search_result = easy_market.search_learnware(user_info)
single_result = search_result.get_single_results()
multiple_result = search_result.get_multiple_results()

print(f"search result of user{idx}:")
print(
f"single model num: {len(single_result)}, max_score: {single_result[0].score}, min_score: {single_result[-1].score}"
)
loss_list = []
for single_item in single_result:
pred_y = single_item.learnware.predict(test_x)
loss_list.append(m5.score(test_y, pred_y))
print(
f"Top1-score: {single_result[0].score}, learnware_id: {single_result[0].learnware.id}, loss: {loss_list[0]}"
)

if len(multiple_result) > 0:
mixture_id = " ".join([learnware.id for learnware in multiple_result[0].learnwares])
print(f"mixture_score: {multiple_result[0].score}, mixture_learnware: {mixture_id}")
mixture_learnware_list = multiple_result[0].learnwares
else:
mixture_learnware_list = [single_result[0].learnware]

reuse_job_selector = JobSelectorReuser(learnware_list=mixture_learnware_list, use_herding=False)
job_selector_predict_y = reuse_job_selector.predict(user_data=test_x)
job_selector_score = m5.score(test_y, job_selector_predict_y)
print(f"mixture reuse loss (job selector): {job_selector_score}")

reuse_ensemble = AveragingReuser(learnware_list=mixture_learnware_list, mode="vote_by_prob")
ensemble_predict_y = reuse_ensemble.predict(user_data=test_x)
ensemble_score = m5.score(test_y, ensemble_predict_y)
print(f"mixture reuse loss (ensemble): {ensemble_score}\n")

single_score_list.append(loss_list[0])
random_score_list.append(np.mean(loss_list))
job_selector_score_list.append(job_selector_score)
ensemble_score_list.append(ensemble_score)
improve_list.append((np.mean(loss_list) - loss_list[0]) / np.mean(loss_list))

logger.info("Single search score %.3f +/- %.3f" % (np.mean(single_score_list), np.std(single_score_list)))
logger.info("Random search score: %.3f +/- %.3f" % (np.mean(random_score_list), np.std(random_score_list)))
logger.info("Average score improvement: %.3f" % (np.mean(improve_list)))
logger.info(
"Job selector score: %.3f +/- %.3f" % (np.mean(job_selector_score_list), np.std(job_selector_score_list))
)
logger.info(
"Average ensemble score: %.3f +/- %.3f" % (np.mean(ensemble_score_list), np.std(ensemble_score_list))
)


if __name__ == "__main__":
fire.Fire(M5DatasetWorkflow)

+ 0
- 87
examples/dataset_m5_workflow/upload.py View File

@@ -1,87 +0,0 @@
import hashlib
import requests
import os
import random
import json
import time
from tqdm import tqdm

email = "tanzh@lamda.nju.edu.cn"
password = hashlib.md5(b"Qwerty123").hexdigest()
login_url = "http://210.28.134.201:8089/auth/login"
submit_url = "http://210.28.134.201:8089/user/add_learnware"
all_data_type = ["Table", "Image", "Video", "Text", "Audio"]
all_task_type = [
"Classification",
"Regression",
"Clustering",
"Feature Extraction",
"Generation",
"Segmentation",
"Object Detection",
]
all_device_type = ["CPU", "GPU"]
all_scenario = [
"Business",
"Financial",
"Health",
"Politics",
"Computer",
"Internet",
"Traffic",
"Nature",
"Fashion",
"Industry",
"Agriculture",
"Education",
"Entertainment",
"Architecture",
]

# ###############
# 以上部分无需修改 #
# ###############


def main():
session = requests.Session()
res = session.post(login_url, json={"email": email, "password": password})

# /path/to/learnware/folder 修改为学件文件夹地址
learnware_pool = os.listdir(os.path.join(os.path.abspath("."), "learnware_pool"))

for learnware in learnware_pool:
# 修改相应的语义规约
name = "M5_Shop" + "%02d" % int(learnware.split(".")[0].split("_")[1])
name = name + "_" + time.strftime("%Y%m%d%H%M%S", time.localtime())
description = f"This is a description of learnware {name}"
data = random.choice(all_data_type)
task = random.choice(all_task_type)
device = list(set(random.choices(all_device_type, k=2)))
scenario = list(set(random.choices(all_scenario, k=5)))
semantic_specification = {
"Data": {"Values": ["Table"], "Type": "Class"},
"Task": {"Values": ["Regression"], "Type": "Class"},
"Device": {"Values": ["CPU"], "Type": "Tag"},
"Scenario": {"Values": ["Business"], "Type": "Tag"},
"Description": {"Values": "A sales-forecasting model from Walmart store", "Type": "String"},
"Name": {"Values": name, "Type": "String"},
"License": {"Values": ["MIT"], "Type": "Class"},
}
res = session.post(
submit_url,
data={
"semantic_specification": json.dumps(semantic_specification),
},
files={
"learnware_file": open(
os.path.join(os.path.abspath("."), "learnware_pool", learnware),
"rb",
)
},
)
assert json.loads(res.text)["code"] == 0, "Upload error"


if __name__ == "__main__":
main()

+ 0
- 8
examples/dataset_pfs_workflow/example.yaml View File

@@ -1,8 +0,0 @@
model:
class_name: Model
kwargs: {}
stat_specifications:
- module_path: learnware.specification
class_name: RKMETableSpecification
file_name: rkme.json
kwargs: {}

+ 0
- 20
examples/dataset_pfs_workflow/example_init.py View File

@@ -1,20 +0,0 @@
import os
import joblib
import numpy as np
from learnware.model import BaseModel


class Model(BaseModel):
def __init__(self):
super(Model, self).__init__(input_shape=(31,), output_shape=(1,))
dir_path = os.path.dirname(os.path.abspath(__file__))
self.model = joblib.load(os.path.join(dir_path, "model.out"))

def fit(self, X: np.ndarray, y: np.ndarray):
pass

def predict(self, X: np.ndarray) -> np.ndarray:
return self.model.predict(X)

def finetune(self, X: np.ndarray, y: np.ndarray):
pass

+ 0
- 208
examples/dataset_pfs_workflow/main.py View File

@@ -1,208 +0,0 @@
import os
import fire
import zipfile
import time
import numpy as np
from tqdm import tqdm
from shutil import copyfile, rmtree

import learnware
from learnware.market import instantiate_learnware_market, BaseUserInfo
from learnware.reuse import JobSelectorReuser, AveragingReuser
from learnware.specification import generate_rkme_table_spec
from pfs import Dataloader
from learnware.logger import get_module_logger

logger = get_module_logger("pfs_test", level="INFO")

output_description = {
"Dimension": 1,
"Description": {},
}

input_description = {
"Dimension": 31,
"Description": {},
}

semantic_specs = [
{
"Data": {"Values": ["Table"], "Type": "Class"},
"Task": {"Values": ["Regression"], "Type": "Class"},
"Library": {"Values": ["Scikit-learn"], "Type": "Class"},
"Scenario": {"Values": ["Business"], "Type": "Tag"},
"Description": {"Values": "", "Type": "String"},
"Name": {"Values": "learnware_1", "Type": "String"},
"Input": input_description,
"Output": output_description,
"License": {"Values": ["MIT"], "Type": "Class"},
}
]

user_semantic = {
"Data": {"Values": ["Table"], "Type": "Class"},
"Task": {"Values": ["Regression"], "Type": "Class"},
"Library": {"Values": ["Scikit-learn"], "Type": "Class"},
"Scenario": {"Values": ["Business"], "Type": "Tag"},
"Description": {"Values": "", "Type": "String"},
"Name": {"Values": "", "Type": "String"},
"Input": input_description,
"Output": output_description,
"License": {"Values": ["MIT"], "Type": "Class"},
}


class PFSDatasetWorkflow:
def _init_pfs_dataset(self):
pfs = Dataloader()
pfs.regenerate_data()

algo_list = ["ridge"] # "ridge", "lgb"
for algo in algo_list:
pfs.set_algo(algo)
pfs.retrain_models()

def _init_learnware_market(self):
"""initialize learnware market"""
learnware.init()
easy_market = instantiate_learnware_market(market_id="pfs", name="easy", rebuild=True)
print("Total Item:", len(easy_market))

zip_path_list = []
curr_root = os.path.dirname(os.path.abspath(__file__))
curr_root = os.path.join(curr_root, "learnware_pool")
for zip_path in os.listdir(curr_root):
zip_path_list.append(os.path.join(curr_root, zip_path))

for idx, zip_path in enumerate(zip_path_list):
semantic_spec = semantic_specs[0]
semantic_spec["Name"]["Values"] = "learnware_%d" % (idx)
semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx)
easy_market.add_learnware(zip_path, semantic_spec)

print("Total Item:", len(easy_market))

def prepare_learnware(self, regenerate_flag=False):
if regenerate_flag:
self._init_pfs_dataset()

pfs = Dataloader()
idx_list = pfs.get_idx_list()
algo_list = ["ridge"] # ["ridge", "lgb"]

curr_root = os.path.dirname(os.path.abspath(__file__))
curr_root = os.path.join(curr_root, "learnware_pool")
os.makedirs(curr_root, exist_ok=True)

for idx in tqdm(idx_list):
train_x, train_y, test_x, test_y = pfs.get_idx_data(idx)
st = time.time()
spec = generate_rkme_table_spec(X=train_x, gamma=0.1, cuda_idx=0)
ed = time.time()
logger.info("Stat spec generated in %.3f s" % (ed - st))

for algo in algo_list:
pfs.set_algo(algo)
dir_path = os.path.join(curr_root, f"{algo}_{idx}")
os.makedirs(dir_path, exist_ok=True)

spec_path = os.path.join(dir_path, "rkme.json")
spec.save(spec_path)

model_path = pfs.get_model_path(idx)
model_file = os.path.join(dir_path, "model.out")
copyfile(model_path, model_file)

init_file = os.path.join(dir_path, "__init__.py")
copyfile("example_init.py", init_file)

yaml_file = os.path.join(dir_path, "learnware.yaml")
copyfile("example.yaml", yaml_file)

zip_file = dir_path + ".zip"
with zipfile.ZipFile(zip_file, "w") as zip_obj:
for foldername, subfolders, filenames in os.walk(dir_path):
for filename in filenames:
file_path = os.path.join(foldername, filename)
zip_info = zipfile.ZipInfo(filename)
zip_info.compress_type = zipfile.ZIP_STORED
with open(file_path, "rb") as file:
zip_obj.writestr(zip_info, file.read())

rmtree(dir_path)

def test(self, regenerate_flag=False):
self.prepare_learnware(regenerate_flag)
self._init_learnware_market()

easy_market = instantiate_learnware_market(market_id="pfs", name="easy")
print("Total Item:", len(easy_market))

pfs = Dataloader()
idx_list = pfs.get_idx_list()
os.makedirs("./user_spec", exist_ok=True)
single_score_list = []
random_score_list = []
job_selector_score_list = []
ensemble_score_list = []
improve_list = []

for idx in idx_list:
train_x, train_y, test_x, test_y = pfs.get_idx_data(idx)
user_spec = generate_rkme_table_spec(X=test_x, gamma=0.1, cuda_idx=0)
user_spec_path = f"./user_spec/user_{idx}.json"
user_spec.save(user_spec_path)

user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": user_spec})
search_result = easy_market.search_learnware(user_info)
single_result = search_result.get_single_results()
multiple_result = search_result.get_multiple_results()

print(f"search result of user{idx}:")
print(
f"single model num: {len(single_result)}, max_score: {single_result[0].score}, min_score: {single_result[-1].score}"
)
loss_list = []
for single_item in single_result:
pred_y = single_item.learnware.predict(test_x)
loss_list.append(pfs.score(test_y, pred_y))
print(
f"Top1-score: {single_result[0].score}, learnware_id: {single_result[0].learnware.id}, loss: {loss_list[0]}, random: {np.mean(loss_list)}"
)

if len(multiple_result) > 0:
mixture_id = " ".join([learnware.id for learnware in multiple_result[0].learnwares])
print(f"mixture_score: {multiple_result[0].score}, mixture_learnware: {mixture_id}")
mixture_learnware_list = multiple_result[0].learnwares
else:
mixture_learnware_list = [single_result[0].learnware]

reuse_job_selector = JobSelectorReuser(learnware_list=mixture_learnware_list, use_herding=False)
job_selector_predict_y = reuse_job_selector.predict(user_data=test_x)
job_selector_score = pfs.score(test_y, job_selector_predict_y)
print(f"mixture reuse loss (job selector): {job_selector_score}")

reuse_ensemble = AveragingReuser(learnware_list=mixture_learnware_list)
ensemble_predict_y = reuse_ensemble.predict(user_data=test_x)
ensemble_score = pfs.score(test_y, ensemble_predict_y)
print(f"mixture reuse loss (ensemble): {ensemble_score}\n")

single_score_list.append(loss_list[0])
random_score_list.append(np.mean(loss_list))
job_selector_score_list.append(job_selector_score)
ensemble_score_list.append(ensemble_score)
improve_list.append((np.mean(loss_list) - loss_list[0]) / np.mean(loss_list))

logger.info("Single search score %.3f +/- %.3f" % (np.mean(single_score_list), np.std(single_score_list)))
logger.info("Random search score: %.3f +/- %.3f" % (np.mean(random_score_list), np.std(random_score_list)))
logger.info("Average score improvement: %.3f" % (np.mean(improve_list)))
logger.info(
"Job selector score: %.3f +/- %.3f" % (np.mean(job_selector_score_list), np.std(job_selector_score_list))
)
logger.info(
"Average ensemble score: %.3f +/- %.3f" % (np.mean(ensemble_score_list), np.std(ensemble_score_list))
)


if __name__ == "__main__":
fire.Fire(PFSDatasetWorkflow)

+ 0
- 48
examples/dataset_pfs_workflow/pfs/README.md View File

@@ -1,48 +0,0 @@
# Learnware based on Prediction Future Sales (PFS) data downloaded from Kaggle
--> Data Page Link: https://www.kaggle.com/c/competitive-data-science-predict-future-sales/data
--> Code Page Link: https://www.kaggle.com/uladzimirkapeika/feature-engineering-lightgbm-top-1
# PFS任务描述
--> 目标：预测每个商店每个商品在下一个月的销量(注意：粒度为月，而不是每天)
--> 特征信息：商店所在城市信息、商品类别信息、商品价格信息、商品历史价格信息(特征工程中只使用了前三个月的历史信息然后拼接在一起)等
--> 使用的模型：XgBoost, LightGBM, LinearRegression
--> 评价指标：RMSE
* split_pfs_data.py
--> 根据Kaggle上公开的数据预处理方案处理下载的数据
--> 直接运行即可将数据根据Shop ID划分为每个商店的信息，包括：
----> 每个商品在每个月下的特征和目标值，存储为pandas.DataFrame格式
----> 字段包括：
-- 标识信息： 'shop_id', 'item_id', 'date_block_num' (标识月份),
-- 目标值(本月销量)： 'item_cnt_month',
-- 城市信息： 'city_code', 'city_coord_1', 'city_coord_2', 'country_part',
-- 商品种类信息： 'item_category_common', 'item_category_code',
-- 该月的时间信息： 'weeknd_count', 'days_in_month',
-- 商品是否第一次销售： 'item_first_interaction', 'shop_item_sold_before',
-- 商品前三个月的销售量和价格信息：
'item_cnt_month_lag_1', 'item_cnt_month_lag_2', 'item_cnt_month_lag_3',
'item_shop_price_avg_lag_1', 'item_shop_price_avg_lag_2', 'item_shop_price_avg_lag_3',
'item_target_enc_lag_1', 'item_target_enc_lag_2', 'item_target_enc_lag_3',
'item_loc_target_enc_lag_1', 'item_loc_target_enc_lag_2', 'item_loc_target_enc_lag_3', 'item_shop_target_enc_lag_1', 'item_shop_target_enc_lag_2', 'item_shop_target_enc_lag_3',
'new_item_cat_avg_lag_1', 'new_item_cat_avg_lag_2', 'new_item_cat_avg_lag_3',
'new_item_shop_cat_avg_lag_1', 'new_item_shop_cat_avg_lag_2', 'new_item_shop_cat_avg_lag_3',
'item_cnt_month_lag_1_adv', 'item_cnt_month_lag_2_adv', 'item_cnt_month_lag_3_adv'
----> 特征： 除了'item_cnt_month'之外的列都当做特征列
----> 目标值： 'item_cnt_month'
----> 时间标识： 'data_block_num'将2013.01到2015.10月的数据标识为0-33，要预测的2015.11月数据为34
--> 存储结果分为两部分： 按照时间划分的train & val，是pandas.DataFrame格式
* pfs_cross_transfer.py
--> 在各自商店训练集上训练一个模型，然后在所有商店的测试集上测试，保存两两预测的RMSE结果，并进行分析
--> 分析包括两部分：(1) 对于一个目标商店，其余源域模型的性能均值，方差，最小值(最好的模型)，最大值，超过均值的源域数目，选择最好模型能够提升的比例等等；(2) HeatMap
--> 需要扩展的方向：(1) LightGBM, Ridge, Xgboost，以及超参数调参；(2) 特征工程去除标识信息，例如shop_id, item_id等等
* data_api.py
--> 后续封装的代码，需继续完善
* packages
--> pip install lightgbm

+ 0
- 77
examples/dataset_pfs_workflow/pfs/__init__.py View File

@@ -1,77 +0,0 @@
import joblib
import os
from sklearn.metrics import mean_squared_error


from .pfs_cross_transfer import *
from .split_data import feature_engineering


class Dataloader:
def __init__(self):
self.algo = "ridge"

def regenerate_data(self):
feature_engineering()

def set_algo(self, algo):
self.algo = algo

def get_algo_list(self):
return ["lgb", "ridge"]

def get_idx_list(self):
return [i for i in range(53)]

def get_idx_data(self, idx):
shop_ids = [i for i in range(60) if i not in [0, 1, 40]]
shop_ids = [i for i in shop_ids if i not in [8, 11, 23, 36]]

fpath = os.path.join(pfs_split_dir, "Shop{:0>2d}-train.csv".format(shop_ids[idx]))
train_xs, train_ys, _, _ = load_pfs_data(fpath)
fpath = os.path.join(pfs_split_dir, "Shop{:0>2d}-val.csv".format(shop_ids[idx]))
test_xs, test_ys, _, _ = load_pfs_data(fpath)
return train_xs, train_ys, test_xs, test_ys

def get_model_path(self, idx):
shop_ids = [i for i in range(60) if i not in [0, 1, 40]]
shop_ids = [i for i in shop_ids if i not in [8, 11, 23, 36]]
return os.path.join(model_dir, "{}_Shop{:0>2d}.out".format(self.algo, shop_ids[idx]))

def retrain_models(self):
algo = self.algo
errs = get_errors(algo=algo)

fpath = os.path.join(pfs_res_dir, "PFS_{}_errs.txt".format(algo))
np.savetxt(fpath, errs.T)

plot_heatmap(errs.T, algo)
weights = np.loadtxt(os.path.join(pfs_res_dir, "PFS_{}_weights.txt".format(algo)))
plot_performance(errs.T, weights, algo)

def retrain_split_models(self):
fpath = os.path.join(pfs_res_dir, "PFS_{}_split_errs_user.txt".format(self.algo))
if os.path.exists(fpath):
return np.loadtxt(fpath)
algo = self.algo
errs = get_split_errs(algo=algo)
fpath = os.path.join(pfs_res_dir, "PFS_{}_split_errs_user.txt".format(algo))
np.savetxt(fpath, errs)
return errs

def get_errs(self):
return np.loadtxt(os.path.join(pfs_res_dir, "PFS_{}_errs.txt".format(self.algo)))

def get_weights(self):
return np.loadtxt(os.path.join(pfs_res_dir, "PFS_{}_weights.txt".format(self.algo)))

def predict(self, idx, test_x):
shop_ids = [i for i in range(60) if i not in [0, 1, 40]]
shop_ids = [i for i in shop_ids if i not in [8, 11, 23, 36]]

model = joblib.load(os.path.join(model_dir, "{}_Shop{:0>2d}.out".format(self.algo, shop_ids[idx])))
# test_x = (test_x - test_x.min(0)) / (test_x.max(0) - test_x.min(0) + 0.0001)
return model.predict(test_x)

def score(self, real_y, pred_y, sample_weight=None):
return mean_squared_error(real_y, pred_y, sample_weight=sample_weight, squared=False)

+ 0
- 272
examples/dataset_pfs_workflow/pfs/config.py View File

@@ -1,272 +0,0 @@
market_store_list = [
0,
2,
3,
4,
5,
6,
7,
8,
9,
10,
12,
13,
14,
15,
16,
17,
18,
20,
22,
23,
24,
25,
26,
27,
28,
30,
31,
32,
33,
34,
35,
37,
38,
39,
40,
42,
44,
45,
46,
47,
48,
50,
52,
]
user_store_list = [1, 11, 19, 21, 29, 36, 43, 49]

dataset_info = {
"name": "PFS",
"range of date": "2014.01-2015.10",
"description": "You are provided with daily historical sales data. The task is to forecast the total amount of products sold in every shop for the test set. Note that the list of shops and products slightly changes every month. More specifically, the dataset involves 53 shops in Russia",
"location_original": [
"Адыгея, Россия",
"Балашиха, Россия",
"Волжский, Россия",
"Вологда, Россия",
"Воронеж, Россия",
"Воронеж, Россия",
"Воронеж, Россия",
"выезд, Россия",
"Жуковский, Россия",
"интернет-магазин, Россия",
"Казань, Россия",
"Калуга, Россия",
"колонна, Россия",
"Красноярск, Россия",
"Красноярск, Россия",
"курск, Россия",
"Москва, Россия",
"Москва, Россия",
"Москва, Россия",
"Москва, Россия",
"Москва, Россия",
"Москва, Россия",
"Москва, Россия",
"Москва, Россия",
"Москва, Россия",
"Москва, Россия",
"Москва, Россия",
"Москва, Россия",
"Мытищи, Россия",
"Н.Новгород, Россия",
"Н.Новгород, Россия",
"Новосибирск, Россия",
"Новосибирск, Россия",
"Ростовнадон, Россия",
"Ростовнадон, Россия",
"спб, Россия",
"спб, Россия",
"самара, Россия",
"самара, Россия",
"Сергий, Россия",
"Сургут, Россия",
"томск, Россия",
"тюмень, Россия",
"тюмень, Россия",
"тюмень, Россия",
"Уфа, Россия",
"Уфа, Россия",
"Химки, Россия",
"цифровой, Россия",
"Чехи, Россия",
"Якутск, Россия",
"Якутск, Россия",
"Ярославль, Россия",
],
"location_english": [
"adygea, Russia",
"Balashikha, Russia",
"Volzhsky, Russia",
"Vologda, Russia",
"Voronezh, Russia",
"Voronezh, Russia",
"Voronezh, Russia",
"outbound, Russia",
"zhukovsky, Russia",
"online stor, Russia",
"Kazan, Russia",
"Kaluga, Russia",
"column, Russia",
"Krasnoyarsk, Russia",
"Krasnoyarsk, Russia",
"kursk, Russia",
"Moscow, Russia",
"Moscow, Russia",
"Moscow, Russia",
"Moscow, Russia",
"Moscow, Russia",
"Moscow, Russia",
"Moscow, Russia",
"Moscow, Russia",
"Moscow, Russia",
"Moscow, Russia",
"Moscow, Russia",
"Moscow, Russia",
"mytishchi, Russia",
"N.Novgorod, Russia",
"N.Novgorod, Russia",
"Novosibirsk, Russia",
"Novosibirsk, Russia",
"rostovnadon, Russia",
"rostovnadon, Russia",
"spb, Russia",
"spb, Russia",
"samara, Russia",
"samara, Russia",
"Sergius, Russia",
"surgut, Russia",
"tomsk, Russia",
"tyumen, Russia",
"tyumen, Russia",
"tyumen, Russia",
"Ufa, Russia",
"Ufa, Russia",
"Khimki, Russia",
"numeric, Russia",
"Czechs, Russia",
"Yakutsk, Russia",
"Yakutsk, Russia",
"Yaroslavl, Russia",
],
"location_chinese": [
"阿迪格亚, 俄罗斯",
"巴拉希哈, 俄罗斯",
"沃尔日斯基, 俄罗斯",
"沃洛格达, 俄罗斯",
"沃罗涅日, 俄罗斯",
"沃罗涅日, 俄罗斯",
"沃罗涅日, 俄罗斯",
"对外贸易, 俄罗斯",
"茹科夫斯基, 俄罗斯",
"在线商店, 俄罗斯",
"喀山, 俄罗斯",
"卡卢加, 俄罗斯",
"科洛姆纳, 俄罗斯",
"克拉斯诺亚尔斯克, 俄罗斯",
"克拉斯诺亚尔斯克, 俄罗斯",
"库尔斯克, 俄罗斯",
"莫斯科, 俄罗斯",
"莫斯科, 俄罗斯",
"莫斯科, 俄罗斯",
"莫斯科, 俄罗斯",
"莫斯科, 俄罗斯",
"莫斯科, 俄罗斯",
"莫斯科, 俄罗斯",
"莫斯科, 俄罗斯",
"莫斯科, 俄罗斯",
"莫斯科, 俄罗斯",
"莫斯科, 俄罗斯",
"莫斯科, 俄罗斯",
"梅季希, 俄罗斯",
"北诺夫哥罗德, 俄罗斯",
"北诺夫哥罗德, 俄罗斯",
"新西伯利亚, 俄罗斯",
"新西伯利亚, 俄罗斯",
"罗斯托夫纳东, 俄罗斯",
"罗斯托夫纳东, 俄罗斯",
"圣彼得堡, 俄罗斯",
"圣彼得堡, 俄罗斯",
"萨马拉, 俄罗斯",
"萨马拉, 俄罗斯",
"谢尔盖, 俄罗斯",
"苏尔古特, 俄罗斯",
"托木斯克, 俄罗斯",
"秋明, 俄罗斯",
"秋明, 俄罗斯",
"秋明, 俄罗斯",
"乌法, 俄罗斯",
"乌法, 俄罗斯",
"希姆基, 俄罗斯",
"在线商店, 俄罗斯",
"契诃夫, 俄罗斯",
"雅库茨克, 俄罗斯",
"雅库茨克, 俄罗斯",
"雅罗斯拉夫尔, 俄罗斯",
],
"memory(KB)": [
246,
302,
3631,
379,
862,
1020,
471,
867,
588,
233,
657,
1272,
801,
469,
146,
1309,
98,
1003,
932,
257,
1959,
1361,
35,
3265,
217,
283,
4311,
1155,
43,
1388,
1971,
971,
7272,
2782,
304,
6801,
4942,
181,
190,
3664,
2061,
170,
807,
593,
1584,
257,
1819,
50,
1063,
692,
336,
277,
743,
],
}

+ 0
- 21
examples/dataset_pfs_workflow/pfs/paths.py View File

@@ -1,21 +0,0 @@
import os
ROOT_PATH = os.path.abspath(os.path.join(__file__, "..", "data"))
raw_data_dir = os.path.join(ROOT_PATH, "raw_data")
split_data_dir = os.path.join(ROOT_PATH, "split_data")
res_dir = os.path.join(ROOT_PATH, "results")
model_dir = os.path.join(ROOT_PATH, "models")
model_dir2 = os.path.join(ROOT_PATH, "models2")
for dir_name in [ROOT_PATH, raw_data_dir, split_data_dir, res_dir, model_dir, model_dir2]:
if not os.path.exists(dir_name):
os.mkdir(dir_name)
pfs_data_dir = os.path.join(raw_data_dir, "PFS")
pfs_split_dir = os.path.join(split_data_dir, "PFS")
pfs_res_dir = os.path.join(res_dir, "PFS")
for dir_name in [pfs_data_dir, pfs_split_dir, pfs_res_dir]:
if not os.path.exists(dir_name):
os.mkdir(dir_name)

+ 0
- 382
examples/dataset_pfs_workflow/pfs/pfs_cross_transfer.py View File

@@ -1,382 +0,0 @@
import os
import pickle
import joblib
import numpy as np
import pandas as pd
import lightgbm as lgb
from sklearn.linear_model import Ridge
from sklearn.model_selection import GridSearchCV
from matplotlib import pyplot as plt
import matplotlib.ticker as ticker
from mpl_toolkits.axes_grid1 import make_axes_locatable
np.seterr(divide="ignore", invalid="ignore")
from .paths import pfs_split_dir, pfs_res_dir, model_dir
np.random.seed(0)
def load_pfs_data(fpath):
df = pd.read_csv(fpath)
features = list(df.columns)
features.remove("item_cnt_month")
features.remove("date_block_num")
# remove id info
# features.remove('shop_id')
# features.remove('item_id')
# remove discrete info
# features.remove('city_code')
# features.remove('item_category_code')
# features.remove('item_category_common')
xs = df[features].values
ys = df["item_cnt_month"].values
categorical_feature_names = ["country_part", "item_category_common", "item_category_code", "city_code"]
types = None
return xs, ys, features, types
def get_split_errs(algo):
"""
according to proportion_list, generate errs whose shape is [shop, split_data]
"""
shop_ids = [i for i in range(60) if i not in [0, 1, 40]]
shop_ids = [i for i in shop_ids if i not in [8, 11, 23, 36]]
user_list = [i for i in range(53)]
proportion_list = [100, 300, 500, 700, 900, 1000, 3000, 5000, 7000, 9000, 10000, 30000, 50000, 70000]
# train
errs = np.zeros((len(user_list), len(proportion_list)))
for s, sid in enumerate(user_list):
# load train data
fpath = os.path.join(pfs_split_dir, "Shop{:0>2d}-train.csv".format(shop_ids[sid]))
fpath_val = os.path.join(pfs_split_dir, "Shop{:0>2d}-val.csv".format(shop_ids[sid]))
train_xs, train_ys, _, _ = load_pfs_data(fpath)
val_xs, val_ys, _, _ = load_pfs_data(fpath_val)
print(shop_ids[sid], train_xs.shape, train_ys.shape)
# data regu
# train_xs = (train_xs - train_xs.min(0)) / (train_xs.max(0) - train_xs.min(0) + 0.0001)
# val_xs = (val_xs - val_xs.min(0)) / (val_xs.max(0) - val_xs.min(0) + 0.0001)
if algo == "lgb":
for tmp in range(len(proportion_list)):
model = lgb.LGBMModel(
boosting_type="gbdt",
num_leaves=2**7 - 1,
learning_rate=0.01,
objective="rmse",
metric="rmse",
feature_fraction=0.75,
bagging_fraction=0.75,
bagging_freq=5,
seed=1,
verbose=1,
n_estimators=100000,
)
model_ori = joblib.load(os.path.join(model_dir, "{}_Shop{:0>2d}.out".format("lgb", shop_ids[sid])))
para = model_ori.get_params()
para["n_estimators"] = 1000
model.set_params(**para)
split = train_xs.shape[0] - proportion_list[tmp]
model.fit(
train_xs[split:,],
train_ys[split:],
eval_set=[(val_xs, val_ys)],
early_stopping_rounds=50,
verbose=100,
)
pred_ys = model.predict(val_xs)
rmse = np.sqrt(((val_ys - pred_ys) ** 2).mean())
errs[s][tmp] = rmse
return errs
def get_errors(algo):
shop_ids = [i for i in range(60) if i not in [0, 1, 40]]
shop_ids = [i for i in shop_ids if i not in [8, 11, 23, 36]]
# train
K = len(shop_ids)
feature_weight = np.zeros(())
errs = np.zeros((K, K))
for s, sid in enumerate(shop_ids):
# load train data
fpath = os.path.join(pfs_split_dir, "Shop{:0>2d}-train.csv".format(sid))
fpath_val = os.path.join(pfs_split_dir, "Shop{:0>2d}-val.csv".format(sid))
train_xs, train_ys, features, _ = load_pfs_data(fpath)
val_xs, val_ys, _, _ = load_pfs_data(fpath_val)
print(sid, train_xs.shape, train_ys.shape)
if s == 0:
feature_weight = np.zeros((K, len(features)))
if algo == "lgb":
model = lgb.LGBMModel(
boosting_type="gbdt",
num_leaves=2**7 - 1,
learning_rate=0.01,
objective="rmse",
metric="rmse",
feature_fraction=0.75,
bagging_fraction=0.75,
bagging_freq=5,
seed=1,
verbose=1,
n_estimators=1000,
)
# train regu data
# train_xs = (train_xs - train_xs.min(0)) / (train_xs.max(0) - train_xs.min(0) + 0.0001)
# val_xs = (val_xs - val_xs.min(0)) / (val_xs.max(0) - val_xs.min(0) + 0.0001)
model.fit(train_xs, train_ys, eval_set=[(val_xs, val_ys)], early_stopping_rounds=100, verbose=100)
# grid search
# para = {'learning_rate': [0.005, 0.01, 0.015], 'num_leaves' : [128, 224, 300], 'max_depth' : [50, 66, 80]}
# grid_search = GridSearchCV(model, para, scoring='neg_mean_squared_error')
# grid_result = grid_search.fit(train_xs, train_ys, eval_set=[(val_xs, val_ys)], verbose = 1000, early_stopping_rounds=1000)
# model = grid_result.best_estimator_
joblib.dump(model, os.path.join(model_dir, "{}_Shop{:0>2d}.out".format(algo, sid)))
importances = model.feature_importances_
elif algo == "ridge":
# train_xs = (train_xs - train_xs.min(0)) / (train_xs.max(0) - train_xs.min(0) + 0.0001)
model = Ridge()
para = {"alpha": [0.01, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10, 20, 30]}
grid_search = GridSearchCV(model, para)
grid_result = grid_search.fit(train_xs, train_ys)
model = grid_result.best_estimator_
importances = model.coef_
joblib.dump(model, os.path.join(model_dir, "{}_Shop{:0>2d}.out".format(algo, sid)))
feature_weight[s] = importances
# leave one out test
for t, tid in enumerate(shop_ids):
# load test data
fpath = os.path.join(pfs_split_dir, "Shop{:0>2d}-val.csv".format(tid))
test_xs, test_ys, _, _ = load_pfs_data(fpath)
# data regu
# test_xs = (test_xs - test_xs.min(0)) / (test_xs.max(0) - test_xs.min(0) + 0.0001)
pred_ys = model.predict(test_xs)
rmse = np.sqrt(((test_ys - pred_ys) ** 2).mean())
print("Shop{} --> Shop{}: {}".format(s, t, rmse))
errs[s][t] = rmse
np.savetxt(os.path.join(pfs_res_dir, "PFS_{}_weights.txt".format(algo)), feature_weight)
return errs
def plot_heatmap(mat, algo):
x_labels = [f"Model{i}" for i in range(mat.shape[1])]
y_labels = [f"Task{i}" for i in range(mat.shape[0])]
fig = plt.figure(figsize=(10, 9))
plt.subplot(1, 1, 1)
ax = plt.gca()
im = plt.imshow(mat)
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="4%", pad=0.3)
plt.colorbar(im, cax=cax)
ax.set_xticks(range(len(x_labels)))
ax.set_xticklabels(x_labels)
ax.set_yticks(range(len(y_labels)))
ax.set_yticklabels(y_labels)
ax.xaxis.set_major_locator(ticker.MultipleLocator(base=5))
ax.yaxis.set_major_locator(ticker.MultipleLocator(base=5))
ax.set_title(f"RMSE on Test set ({algo})")
plt.tight_layout()
plt.savefig(os.path.join(pfs_res_dir, "PFS_{}_heatmap.jpg".format(algo)), dpi=700)
def plot_var(errs, algo):
avg_err = []
min_err = []
med_err = []
max_err = []
std_err = []
cnts = []
improves = []
for j in range(len(errs)):
inds = [i for i in range(len(errs)) if i != j]
ys = errs[:, j][inds]
avg_err.append(np.mean(ys))
min_err.append(np.min(ys))
med_err.append(np.median(ys))
max_err.append(np.max(ys))
std_err.append(np.std(ys))
cnts.append(np.sum(ys >= np.mean(ys)))
improves.append((np.mean(ys) - np.min(ys)) / np.mean(ys))
avg_err = np.array(avg_err)
min_err = np.array(min_err)
med_err = np.array(med_err)
max_err = np.array(max_err)
std_err = np.array(std_err)
cnts = np.array(cnts)
improves = np.array(improves)
inds = np.argsort(avg_err)
avg_err = avg_err[inds]
min_err = min_err[inds]
med_err = med_err[inds]
max_err = max_err[inds]
std_err = std_err[inds]
cnts = cnts[inds]
improves = improves[inds]
xs = list(range(len(inds)))
fig = plt.figure(figsize=(8, 8))
ax = plt.subplot(3, 1, 1)
ax.plot(xs, avg_err, color="red", linestyle="solid", linewidth=2.5)
ax.plot(xs, min_err, color="blue", linestyle="dotted", linewidth=1.5)
ax.plot(xs, med_err, color="purple", linestyle="solid", linewidth=1.0)
ax.plot(xs, max_err, color="green", linestyle="dashed", linewidth=1.5)
ax.legend(["Avg", "Min", "Median", "Max"], fontsize=14)
ax.fill_between(xs, avg_err - std_err, avg_err + std_err, alpha=0.2)
gap = np.mean(avg_err - min_err)
ax.set_ylabel("RMSE", fontsize=14)
ax.set_title("RMSE of Source Models ({}) [Avg-Min:{:.3f}]".format(algo, gap), fontsize=18)
ax = plt.subplot(3, 1, 2)
ax.bar(xs, cnts)
ax.set_ylabel("Number", fontsize=14)
ax.set_title("Number of sources above average", fontsize=18)
ax = plt.subplot(3, 1, 3)
ax.plot(xs, improves)
ax.set_xlabel("Sorted Shop ID by Avg.Err", fontsize=14)
ax.set_ylabel("Ratio", fontsize=14)
ax.set_title("Best Improve Ratio: (Avg - Min) / Avg", fontsize=18)
fig.tight_layout()
fig.savefig(os.path.join(pfs_res_dir, "{}-var.jpg".format(algo)))
plt.show()
def plot_performance(errs, weights, algo):
avg_err = []
min_err = []
med_err = []
max_err = []
std_err = []
cnts = []
improves = []
for i in range(errs.shape[0]):
inds = [j for j in range(errs.shape[1]) if j != i]
arr = errs[i][inds]
avg_err.append(np.mean(arr))
min_err.append(np.min(arr))
med_err.append(np.median(arr))
max_err.append(np.max(arr))
std_err.append(np.std(arr))
cnts.append(np.sum(arr >= np.mean(arr)))
improves.append((np.mean(arr) - np.min(arr)) / np.mean(arr))
avg_err = np.array(avg_err)
min_err = np.array(min_err)
med_err = np.array(med_err)
max_err = np.array(max_err)
std_err = np.array(std_err)
cnts = np.array(cnts)
improves = np.array(improves)
inds = np.argsort(avg_err)
avg_err = avg_err[inds]
min_err = min_err[inds]
med_err = med_err[inds]
max_err = max_err[inds]
std_err = std_err[inds]
cnts = cnts[inds]
improves = improves[inds]
xs = list(range(len(inds)))
fig = plt.figure(figsize=(12, 9))
ax = plt.subplot(2, 2, 1)
ax.plot(xs, avg_err, color="red", linestyle="solid", linewidth=2.5)
ax.plot(xs, min_err, color="blue", linestyle="dotted", linewidth=1.5)
ax.plot(xs, med_err, color="purple", linestyle="solid", linewidth=1.0)
ax.plot(xs, max_err, color="green", linestyle="dashed", linewidth=1.5)
ax.legend(["Avg", "Min", "Median", "Max"], fontsize=14)
ax.fill_between(xs, avg_err - std_err, avg_err + std_err, alpha=0.2)
gap = np.mean(avg_err - min_err)
ax.set_ylabel("RMSE", fontsize=14)
ax.set_title("RMSE of Source Models ({}) [Avg-Min:{:.3f}]".format(algo, gap), fontsize=18)
ax = plt.subplot(2, 2, 2)
ax.bar(xs, cnts)
ax.set_ylabel("Number", fontsize=14)
ax.set_title("Number of sources above average", fontsize=18)
ax = plt.subplot(2, 2, 3)
ax.plot(xs, improves)
ax.set_xlabel("Sorted Shop ID by Avg.Err", fontsize=14)
ax.set_ylabel("Ratio", fontsize=14)
ax.set_title("Best Improve Ratio: (Avg - Min) / Avg", fontsize=18)
ax = plt.subplot(2, 2, 4)
weights = np.mean(weights, axis=0) / weights.sum()
weights = np.sort(weights)
xs = list(range(len(weights)))
ax.plot(xs, weights)
# ax.set_xlabel("Sorted Feature ID by Avg.Feature_Importance", fontsize=14)
ax.set_ylabel("Proportion", fontsize=14)
ax.set_title("Avg.Feature_Importances", fontsize=18)
fig.tight_layout()
fig.savefig(os.path.join(pfs_res_dir, "PFS_{}_performance.png".format(algo)), dpi=700)
# fig.savefig(f"{algo}_performance.png", dpi=700)
plt.show()
if __name__ == "__main__":
# for algo in ["ridge", "lgb", "xgboost_125"]:
for algo in ["ridge"]:
fpath = os.path.join(pfs_res_dir, "{}_errs.pkl".format(algo))
if os.path.exists(fpath):
with open(fpath, "rb") as fr:
errs = pickle.load(fr)
else:
errs = get_errors(algo=algo)
with open(fpath, "wb") as fw:
pickle.dump(errs, fw)
index = ["Source{}".format(k) for k in range(len(errs))]
columns = ["Target{}".format(k) for k in range(len(errs[0]))]
df = pd.DataFrame(errs, index=index, columns=columns)
fpath = os.path.join(pfs_res_dir, "PFS_{}_errs.txt".format(algo))
# df.to_csv(fpath, index=True)
np.savetxt(fpath, errs.T)
# plot_var(errs, algo)
plot_heatmap(errs.T, algo)
weights = np.loadtxt(os.path.join(pfs_res_dir, "PFS_{}_weights.txt".format(algo)))
plot_performance(errs.T, weights, algo)

+ 0
- 384
examples/dataset_pfs_workflow/pfs/split_data.py View File

@@ -1,384 +0,0 @@
import os
import pickle
import pandas as pd
import numpy as np
from itertools import product
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import MinMaxScaler
import calendar
from .paths import pfs_data_dir
from .paths import pfs_split_dir
def feature_engineering():
# read data
sales = pd.read_csv(os.path.join(pfs_data_dir, "sales_train.csv"))
shops = pd.read_csv(os.path.join(pfs_data_dir, "shops.csv"))
items = pd.read_csv(os.path.join(pfs_data_dir, "items.csv"))
item_cats = pd.read_csv(os.path.join(pfs_data_dir, "item_categories.csv"))
test = pd.read_csv(os.path.join(pfs_data_dir, "test.csv"))
# remove outliers
train = sales[(sales.item_price < 10000) & (sales.item_price > 0)]
train = train[sales.item_cnt_day < 1001]
print(train.shape, sales.shape)
print(train.tail(5))
print(sales.tail(5))
# combine shops with different id but the same name
train.loc[train.shop_id == 0, "shop_id"] = 57
test.loc[test.shop_id == 0, "shop_id"] = 57
train.loc[train.shop_id == 1, "shop_id"] = 58
test.loc[test.shop_id == 1, "shop_id"] = 58
train.loc[train.shop_id == 40, "shop_id"] = 39
test.loc[test.shop_id == 40, "shop_id"] = 39
# obtain shop_id, item_id, month information
index_cols = ["shop_id", "item_id", "date_block_num"]
df = []
for block_num in train["date_block_num"].unique():
cur_shops = train.loc[sales["date_block_num"] == block_num, "shop_id"].unique()
cur_items = train.loc[sales["date_block_num"] == block_num, "item_id"].unique()
df.append(np.array(list(product(*[cur_shops, cur_items, [block_num]])), dtype="int32"))
df = pd.DataFrame(np.vstack(df), columns=index_cols, dtype=np.int32)
print("df.shape: ", df.shape)
print(df.head(5))
# Add month sales
group = train.groupby(["date_block_num", "shop_id", "item_id"]).agg({"item_cnt_day": ["sum"]})
group.columns = ["item_cnt_month"]
group.reset_index(inplace=True)
print("group.shape: ", group.shape)
print(group.head(5))
df = pd.merge(df, group, on=index_cols, how="left")
df["item_cnt_month"] = (
df["item_cnt_month"]
.fillna(0)
.astype(np.float32)
# df['item_cnt_month'].fillna(0).clip(0, 20).astype(np.float32)
)
# fill test data
test["date_block_num"] = 34
test["date_block_num"] = test["date_block_num"].astype(np.int8)
test["shop_id"] = test["shop_id"].astype(np.int8)
test["item_id"] = test["item_id"].astype(np.int16)
df = pd.concat([df, test], ignore_index=True, sort=False, keys=index_cols)
df.fillna(0, inplace=True)
# shop location features
shops["city"] = shops["shop_name"].apply(lambda x: x.split()[0].lower())
shops.loc[shops.city == "!якутск", "city"] = "якутск"
shops["city_code"] = LabelEncoder().fit_transform(shops["city"])
coords = dict()
coords["якутск"] = (62.028098, 129.732555, 4)
coords["адыгея"] = (44.609764, 40.100516, 3)
coords["балашиха"] = (55.8094500, 37.9580600, 1)
coords["волжский"] = (53.4305800, 50.1190000, 3)
coords["вологда"] = (59.2239000, 39.8839800, 2)
coords["воронеж"] = (51.6720400, 39.1843000, 3)
coords["выездная"] = (0, 0, 0)
coords["жуковский"] = (55.5952800, 38.1202800, 1)
coords["интернет-магазин"] = (0, 0, 0)
coords["казань"] = (55.7887400, 49.1221400, 4)
coords["калуга"] = (54.5293000, 36.2754200, 4)
coords["коломна"] = (55.0794400, 38.7783300, 4)
coords["красноярск"] = (56.0183900, 92.8671700, 4)
coords["курск"] = (51.7373300, 36.1873500, 3)
coords["москва"] = (55.7522200, 37.6155600, 1)
coords["мытищи"] = (55.9116300, 37.7307600, 1)
coords["н.новгород"] = (56.3286700, 44.0020500, 4)
coords["новосибирск"] = (55.0415000, 82.9346000, 4)
coords["омск"] = (54.9924400, 73.3685900, 4)
coords["ростовнадону"] = (47.2313500, 39.7232800, 3)
coords["спб"] = (59.9386300, 30.3141300, 2)
coords["самара"] = (53.2000700, 50.1500000, 4)
coords["сергиев"] = (56.3000000, 38.1333300, 4)
coords["сургут"] = (61.2500000, 73.4166700, 4)
coords["томск"] = (56.4977100, 84.9743700, 4)
coords["тюмень"] = (57.1522200, 65.5272200, 4)
coords["уфа"] = (54.7430600, 55.9677900, 4)
coords["химки"] = (55.8970400, 37.4296900, 1)
coords["цифровой"] = (0, 0, 0)
coords["чехов"] = (55.1477000, 37.4772800, 4)
coords["ярославль"] = (57.6298700, 39.8736800, 2)
shops["city_coord_1"] = shops["city"].apply(lambda x: coords[x][0])
shops["city_coord_2"] = shops["city"].apply(lambda x: coords[x][1])
shops["country_part"] = shops["city"].apply(lambda x: coords[x][2])
shops = shops[["shop_id", "city_code", "city_coord_1", "city_coord_2", "country_part"]]
df = pd.merge(df, shops, on=["shop_id"], how="left")
# process items category name
map_dict = {
"Чистые носители (штучные)": "Чистые носители",
"Чистые носители (шпиль)": "Чистые носители",
"PC ": "Аксессуары",
"Служебные": "Служебные ",
}
items = pd.merge(items, item_cats, on="item_category_id")
items["item_category"] = items["item_category_name"].apply(lambda x: x.split("-")[0])
items["item_category"] = items["item_category"].apply(lambda x: map_dict[x] if x in map_dict.keys() else x)
items["item_category_common"] = LabelEncoder().fit_transform(items["item_category"])
items["item_category_code"] = LabelEncoder().fit_transform(items["item_category_name"])
items = items[["item_id", "item_category_common", "item_category_code"]]
df = pd.merge(df, items, on=["item_id"], how="left")
# Weekends count / number of days in a month
def count_days(date_block_num):
year = 2013 + date_block_num // 12
month = 1 + date_block_num % 12
weeknd_count = len([1 for i in calendar.monthcalendar(year, month) if i[6] != 0])
days_in_month = calendar.monthrange(year, month)[1]
return weeknd_count, days_in_month, month
map_dict = {i: count_days(i) for i in range(35)}
df["weeknd_count"] = df["date_block_num"].apply(lambda x: map_dict[x][0])
df["days_in_month"] = df["date_block_num"].apply(lambda x: map_dict[x][1])
# Interation features: Item is new / Item was bought in this shop before
first_item_block = df.groupby(["item_id"])["date_block_num"].min().reset_index()
first_item_block["item_first_interaction"] = 1
first_shop_item_buy_block = (
df[df["date_block_num"] > 0].groupby(["shop_id", "item_id"])["date_block_num"].min().reset_index()
)
first_shop_item_buy_block["first_date_block_num"] = first_shop_item_buy_block["date_block_num"]
df = pd.merge(
df,
first_item_block[["item_id", "date_block_num", "item_first_interaction"]],
on=["item_id", "date_block_num"],
how="left",
)
df = pd.merge(
df,
first_shop_item_buy_block[["item_id", "shop_id", "first_date_block_num"]],
on=["item_id", "shop_id"],
how="left",
)
df["first_date_block_num"].fillna(100, inplace=True)
df["shop_item_sold_before"] = (df["first_date_block_num"] < df["date_block_num"]).astype("int8")
df.drop(["first_date_block_num"], axis=1, inplace=True)
df["item_first_interaction"].fillna(0, inplace=True)
df["shop_item_sold_before"].fillna(0, inplace=True)
df["item_first_interaction"] = df["item_first_interaction"].astype("int8")
df["shop_item_sold_before"] = df["shop_item_sold_before"].astype("int8")
def lag_feature(df, lags, col):
tmp = df[["date_block_num", "shop_id", "item_id", col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ["date_block_num", "shop_id", "item_id", col + "_lag_" + str(i)]
shifted["date_block_num"] += i
df = pd.merge(df, shifted, on=["date_block_num", "shop_id", "item_id"], how="left")
lag_name = col + "_lag_" + str(i)
df[lag_name] = df[lag_name].astype("float32")
return df
df = lag_feature(df, [1, 2, 3], "item_cnt_month")
index_cols = ["shop_id", "item_id", "date_block_num"]
group = (
train.groupby(index_cols)["item_price"]
.mean()
.reset_index()
.rename(columns={"item_price": "avg_shop_price"}, errors="raise")
)
df = pd.merge(df, group, on=index_cols, how="left")
df["avg_shop_price"] = df["avg_shop_price"].fillna(0).astype(np.float32)
index_cols = ["item_id", "date_block_num"]
group = (
train.groupby(["date_block_num", "item_id"])["item_price"]
.mean()
.reset_index()
.rename(columns={"item_price": "avg_item_price"}, errors="raise")
)
df = pd.merge(df, group, on=index_cols, how="left")
df["avg_item_price"] = df["avg_item_price"].fillna(0).astype(np.float32)
df["item_shop_price_avg"] = (df["avg_shop_price"] - df["avg_item_price"]) / df["avg_item_price"]
df["item_shop_price_avg"].fillna(0, inplace=True)
df = lag_feature(df, [1, 2, 3], "item_shop_price_avg")
df.drop(["avg_shop_price", "avg_item_price", "item_shop_price_avg"], axis=1, inplace=True)
item_id_target_mean = (
df.groupby(["date_block_num", "item_id"])["item_cnt_month"]
.mean()
.reset_index()
.rename(columns={"item_cnt_month": "item_target_enc"}, errors="raise")
)
df = pd.merge(df, item_id_target_mean, on=["date_block_num", "item_id"], how="left")
df["item_target_enc"] = df["item_target_enc"].fillna(0).astype(np.float32)
df = lag_feature(df, [1, 2, 3], "item_target_enc")
df.drop(["item_target_enc"], axis=1, inplace=True)
item_id_target_mean = (
df.groupby(["date_block_num", "item_id", "city_code"])["item_cnt_month"]
.mean()
.reset_index()
.rename(columns={"item_cnt_month": "item_loc_target_enc"}, errors="raise")
)
df = pd.merge(df, item_id_target_mean, on=["date_block_num", "item_id", "city_code"], how="left")
df["item_loc_target_enc"] = df["item_loc_target_enc"].fillna(0).astype(np.float32)
df = lag_feature(df, [1, 2, 3], "item_loc_target_enc")
df.drop(["item_loc_target_enc"], axis=1, inplace=True)
item_id_target_mean = (
df.groupby(["date_block_num", "item_id", "shop_id"])["item_cnt_month"]
.mean()
.reset_index()
.rename(columns={"item_cnt_month": "item_shop_target_enc"}, errors="raise")
)
df = pd.merge(df, item_id_target_mean, on=["date_block_num", "item_id", "shop_id"], how="left")
df["item_shop_target_enc"] = df["item_shop_target_enc"].fillna(0).astype(np.float32)
df = lag_feature(df, [1, 2, 3], "item_shop_target_enc")
df.drop(["item_shop_target_enc"], axis=1, inplace=True)
item_id_target_mean = (
df[df["item_first_interaction"] == 1]
.groupby(["date_block_num", "item_category_code"])["item_cnt_month"]
.mean()
.reset_index()
.rename(columns={"item_cnt_month": "new_item_cat_avg"}, errors="raise")
)
df = pd.merge(df, item_id_target_mean, on=["date_block_num", "item_category_code"], how="left")
df["new_item_cat_avg"] = df["new_item_cat_avg"].fillna(0).astype(np.float32)
df = lag_feature(df, [1, 2, 3], "new_item_cat_avg")
df.drop(["new_item_cat_avg"], axis=1, inplace=True)
# For new items add avg category sales in a separate store for last 3 months
item_id_target_mean = (
df[df["item_first_interaction"] == 1]
.groupby(["date_block_num", "item_category_code", "shop_id"])["item_cnt_month"]
.mean()
.reset_index()
.rename(columns={"item_cnt_month": "new_item_shop_cat_avg"}, errors="raise")
)
df = pd.merge(df, item_id_target_mean, on=["date_block_num", "item_category_code", "shop_id"], how="left")
df["new_item_shop_cat_avg"] = df["new_item_shop_cat_avg"].fillna(0).astype(np.float32)
df = lag_feature(df, [1, 2, 3], "new_item_shop_cat_avg")
df.drop(["new_item_shop_cat_avg"], axis=1, inplace=True)
def lag_feature_adv(df, lags, col):
tmp = df[["date_block_num", "shop_id", "item_id", col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ["date_block_num", "shop_id", "item_id", col + "_lag_" + str(i) + "_adv"]
shifted["date_block_num"] += i
shifted["item_id"] -= 1
df = pd.merge(df, shifted, on=["date_block_num", "shop_id", "item_id"], how="left")
lag_name = col + "_lag_" + str(i) + "_adv"
df[lag_name] = df[lag_name].astype("float32")
return df
df = lag_feature_adv(df, [1, 2, 3], "item_cnt_month")
# df.fillna(0, inplace=True)
df = df[(df["date_block_num"] > 2)]
df.drop(["ID"], axis=1, inplace=True, errors="ignore")
print(df.shape)
print(df.columns)
print(df.head(10))
fill_dict = {}
for col in df.columns:
fill_dict[col] = df[col].mean()
group_df = df.groupby(["shop_id"])
for shop_id, shop_df in group_df:
# remove data of data_block_num=34, i.e., 2015.11
# this is test set in competition
shop_df = shop_df[shop_df.date_block_num <= 33]
# fill the null
cols = shop_df.isnull().any()
idx = list(cols[cols.values].index)
shop_df[idx] = shop_df.groupby("item_id", sort=False)[idx].apply(
lambda x: x.fillna(method="ffill").fillna(method="bfill")
)
shop_df[idx] = shop_df[idx].fillna(shop_df[idx].mean())
for col in idx:
shop_df[col] = shop_df[col].fillna(fill_dict[col])
# min-max scale
drop_fea_list = [
"shop_id",
"city_code",
"city_coord_1",
"city_coord_2",
"country_part",
"item_cnt_month",
"date_block_num",
]
fea_list = [col for col in shop_df.columns if col not in drop_fea_list]
mms = MinMaxScaler()
shop_df[fea_list] = mms.fit_transform(shop_df[fea_list])
shop_df = shop_df[fea_list + ["item_cnt_month", "date_block_num"]]
date_split = 29
split = False
while split is False:
df1 = shop_df[shop_df["date_block_num"] <= date_split]
df2 = shop_df[shop_df["date_block_num"] > date_split]
if df2.shape[0] > 0 and df1.shape[0] > 0:
split = True
else:
date_split -= 1
if date_split < 0:
break
if split is True:
print("ShopID:{}, split block:{}".format(shop_id, date_split))
print(df1.shape, df2.shape)
# save train csv
fpath = os.path.join(pfs_split_dir, "Shop{:0>2d}-train.csv".format(shop_id))
df1.to_csv(fpath, index=False)
# save val csv
fpath = os.path.join(pfs_split_dir, "Shop{:0>2d}-val.csv".format(shop_id))
df2.to_csv(fpath, index=False)

+ 0
- 90
examples/dataset_pfs_workflow/upload.py View File

@@ -1,90 +0,0 @@
import hashlib
import requests
import os
import random
import json
import time
from tqdm import tqdm

email = "liujd@lamda.nju.edu.cn"
password = hashlib.md5(b"liujdlamda").hexdigest()
login_url = "http://210.28.134.201:8089/auth/login"
submit_url = "http://210.28.134.201:8089/user/add_learnware"
all_data_type = ["Table", "Image", "Video", "Text", "Audio"]
all_task_type = [
"Classification",
"Regression",
"Clustering",
"Feature Extraction",
"Generation",
"Segmentation",
"Object Detection",
]
all_device_type = ["CPU", "GPU"]
all_scenario = [
"Business",
"Financial",
"Health",
"Politics",
"Computer",
"Internet",
"Traffic",
"Nature",
"Fashion",
"Industry",
"Agriculture",
"Education",
"Entertainment",
"Architecture",
]

# ###############
# 以上部分无需修改 #
# ###############


def main():
session = requests.Session()
res = session.post(login_url, json={"email": email, "password": password})

# /path/to/learnware/folder 修改为学件文件夹地址
learnware_pool = os.listdir(os.path.join(os.path.abspath("."), "learnware_pool"))

for learnware in learnware_pool:
# 修改相应的语义规约
name = "PFS_Shop" + "%02d" % int(learnware.split(".")[0].split("_")[1])
name = name + "_" + time.strftime("%Y%m%d%H%M%S", time.localtime())
description = f"This is a description of learnware {name}"
data = random.choice(all_data_type)
task = random.choice(all_task_type)
device = list(set(random.choices(all_device_type, k=2)))
scenario = list(set(random.choices(all_scenario, k=5)))
semantic_specification = {
"Data": {"Values": ["Table"], "Type": "Class"},
"Library": {"Values": ["Scikit-learn"], "Type": "Class"},
"Task": {"Values": ["Regression"], "Type": "Class"},
"Scenario": {"Values": ["Business"], "Type": "Tag"},
"Description": {
"Values": "A sales-forecasting model from Predict Future Sales Competition on Kaggle",
"Type": "String",
},
"Name": {"Values": name, "Type": "String"},
"License": {"Values": ["MIT"], "Type": "Class"},
}
res = session.post(
submit_url,
data={
"semantic_specification": json.dumps(semantic_specification),
},
files={
"learnware_file": open(
os.path.join(os.path.abspath("."), "learnware_pool", learnware),
"rb",
)
},
)
assert json.loads(res.text)["code"] == 0, "Upload error"


if __name__ == "__main__":
main()

Write Preview
Loading…
Cancel
Save