svm.py

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.svm import LinearSVC
from sklearn.multioutput import MultiOutputClassifier
from sklearn.metrics import classification_report
from sklearn.preprocessing import MultiLabelBinarizer
import joblib
import ast

df = pd.read_csv("data/questions_tags_embedded.csv")
df['languages'] = df['languages'].apply(ast.literal_eval)
df['target'] = df['target'].apply(ast.literal_eval)

# convert embedding string -> numpy array
X = np.array(df['embedded_question'].apply(lambda x:np.fromstring(x.strip('[]'), sep=' ')).tolist())

mlb = MultiLabelBinarizer()
Y = mlb.fit_transform(df['target'])

# train-val-test split
X_train, X_temp, Y_train, Y_temp = train_test_split(X, Y, test_size=0.2, random_state=42)
X_val, X_test, Y_val, Y_test = train_test_split(X_temp, Y_temp, test_size=0.5, random_state=42)

# train
print("training svm")
base_svm = LinearSVC(random_state=42)
model = MultiOutputClassifier(base_svm)
model.fit(X_train, Y_train)

# test
Y_val_pred = model.predict(X_val)
Y_test_pred = model.predict(X_test)

print("Validation:")
print(classification_report(Y_val, Y_val_pred, target_names=mlb.classes_, zero_division=0))

print("Test:")
print(classification_report(Y_test, Y_test_pred, target_names=mlb.classes_, zero_division=0))

# save model
joblib.dump({"model": model, "mlb": mlb}, "svm_multilabel_classifier.pkl")

print("Model saved as svm_multilabel_classifier.pkl")