decision_tree.py

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier
from sklearn.multioutput import MultiOutputClassifier
from sklearn.metrics import classification_report
import ast
from sklearn.preprocessing import MultiLabelBinarizer

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import classification_report
import ast

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier
from sklearn.multioutput import MultiOutputClassifier
from sklearn.metrics import classification_report
import ast
from sklearn.preprocessing import MultiLabelBinarizer


df = pd.read_csv(r"C:\Users\jiag2\College\asdfa]\questions_tags_embedded.csv")
df['languages'] = df['languages'].apply(ast.literal_eval)
df['target'] = df['target'].apply(ast.literal_eval)
print(df.head(5))

X = np.array(df['embedded_question'].apply(lambda x: np.fromstring(x.strip('[]'), sep=' ')).tolist())
print(X.shape)

mlb = MultiLabelBinarizer()
# Use df['languagues'] for programming language classification
# Y = mlb.fit_transform(df['languages'])

# Use df['target'] for multi label tag classification
Y = mlb.fit_transform(df['target'])

print("Classes:", mlb.classes_)
print("Shape of Y:", Y.shape)

X_train, X_temp, Y_train, Y_temp = train_test_split(X, Y, test_size=0.2, random_state=42)
X_val, X_test, Y_val, Y_test = train_test_split(X_temp, Y_temp, test_size=0.5, random_state=42)

print(X_train.shape, X_val.shape, X_test.shape)

tree = DecisionTreeClassifier(max_depth=10, random_state=42)
model = MultiOutputClassifier(tree, n_jobs=16)

print("Training started...")
model.fit(X_train, Y_train)

Y_val_pred = model.predict(X_val)
Y_test_pred = model.predict(X_test)

print("Validation:")
print(classification_report(Y_val, Y_val_pred, target_names=mlb.classes_, zero_division=0))

print("Test:")
print(classification_report(Y_test, Y_test_pred, target_names=mlb.classes_, zero_division=0))