diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b4aad9943..a4a66815b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,17 +25,17 @@ jobs: - name: Lint with flake8 run: | - flake8 day5/演習3 --count --select=E9,F63,F7,F82 --show-source --statistics - flake8 day5/演習3 --count --exit-zero --max-complexity=10 --max-line-length=88 --statistics + flake8 day5/practice3 --count --select=E9,F63,F7,F82 --show-source --statistics + flake8 day5/practice3 --count --exit-zero --max-complexity=10 --max-line-length=88 --statistics - name: Format check with black run: | - black --check day5/演習3 + black --check day5/practice3 - name: Run data tests run: | - pytest day5/演習3/tests/test_data.py -v + pytest day5/practice3/tests/test_data.py -v - name: Run model tests run: | - pytest day5/演習3/tests/test_model.py -v + pytest day5/practice3/tests/test_model.py -v diff --git "a/day5/\346\274\224\347\277\2221/data/Titanic.csv" b/day5/practice1/data/Titanic.csv similarity index 100% rename from "day5/\346\274\224\347\277\2221/data/Titanic.csv" rename to day5/practice1/data/Titanic.csv diff --git "a/day5/\346\274\224\347\277\2221/main.py" b/day5/practice1/main.py similarity index 74% rename from "day5/\346\274\224\347\277\2221/main.py" rename to day5/practice1/main.py index 08be7c6ab..e9ddc05b4 100644 --- "a/day5/\346\274\224\347\277\2221/main.py" +++ b/day5/practice1/main.py @@ -10,6 +10,7 @@ from sklearn.metrics import accuracy_score from sklearn.preprocessing import LabelEncoder from mlflow.models.signature import infer_signature +import time # データ準備 @@ -76,6 +77,14 @@ def log_model(model, accuracy, params): print(f"モデルのログ記録値 \naccuracy: {accuracy}\nparams: {params}") +def evaluate_inference_time_and_accuracy(model, X_test, y_test): + start_time = time.time() + predictions = model.predict(X_test) + inference_time = time.time() - start_time + accuracy = accuracy_score(y_test, predictions) + return inference_time, accuracy + + # メイン処理 if __name__ == "__main__": # ランダム要素の設定 @@ -121,3 +130,21 @@ def log_model(model, accuracy, params): with open(model_path, "wb") as f: pickle.dump(model, f) print(f"モデルを {model_path} に保存しました") + + # モデルの推論精度と時間を評価 + inference_time, current_accuracy = evaluate_inference_time_and_accuracy(model, X_test, y_test) + mlflow.log_metric("inference_time", inference_time) + mlflow.log_metric("current_accuracy", current_accuracy) + + # 過去バージョンのモデルと比較 + past_model_path = os.path.join(model_dir, "titanic_model.pkl") + if os.path.exists(past_model_path): + with open(past_model_path, "rb") as f: + past_model = pickle.load(f) + past_inference_time, past_accuracy = evaluate_inference_time_and_accuracy(past_model, X_test, y_test) + mlflow.log_metric("past_inference_time", past_inference_time) + mlflow.log_metric("past_accuracy", past_accuracy) + print(f"過去モデルの精度: {past_accuracy}, 推論時間: {past_inference_time}") + print(f"現在モデルの精度: {current_accuracy}, 推論時間: {inference_time}") + else: + print("過去モデルが見つかりませんでした。") diff --git a/day5/practice1/models/titanic_model.pkl b/day5/practice1/models/titanic_model.pkl new file mode 100644 index 000000000..f12c83f17 Binary files /dev/null and b/day5/practice1/models/titanic_model.pkl differ diff --git "a/day5/\346\274\224\347\277\2221/pipeline.py" b/day5/practice1/pipeline.py similarity index 100% rename from "day5/\346\274\224\347\277\2221/pipeline.py" rename to day5/practice1/pipeline.py diff --git "a/day5/\346\274\224\347\277\2222/black_check.py" b/day5/practice2/black_check.py similarity index 100% rename from "day5/\346\274\224\347\277\2222/black_check.py" rename to day5/practice2/black_check.py diff --git "a/day5/\346\274\224\347\277\2222/data/Titanic.csv" b/day5/practice2/data/Titanic.csv similarity index 100% rename from "day5/\346\274\224\347\277\2222/data/Titanic.csv" rename to day5/practice2/data/Titanic.csv diff --git "a/day5/\346\274\224\347\277\2222/data/Titanic_error.csv" b/day5/practice2/data/Titanic_error.csv similarity index 100% rename from "day5/\346\274\224\347\277\2222/data/Titanic_error.csv" rename to day5/practice2/data/Titanic_error.csv diff --git "a/day5/\346\274\224\347\277\2222/main.py" b/day5/practice2/main.py similarity index 100% rename from "day5/\346\274\224\347\277\2222/main.py" rename to day5/practice2/main.py diff --git "a/day5/\346\274\224\347\277\2222/models/titanic_model.pkl" b/day5/practice2/models/titanic_model.pkl similarity index 100% rename from "day5/\346\274\224\347\277\2222/models/titanic_model.pkl" rename to day5/practice2/models/titanic_model.pkl diff --git "a/day5/\346\274\224\347\277\2223/data/Titanic.csv" b/day5/practice3/data/Titanic.csv similarity index 100% rename from "day5/\346\274\224\347\277\2223/data/Titanic.csv" rename to day5/practice3/data/Titanic.csv diff --git "a/day5/\346\274\224\347\277\2223/models/titanic_model.pkl" b/day5/practice3/models/titanic_model.pkl similarity index 100% rename from "day5/\346\274\224\347\277\2223/models/titanic_model.pkl" rename to day5/practice3/models/titanic_model.pkl diff --git "a/day5/\346\274\224\347\277\2223/tests/test_data.py" b/day5/practice3/tests/test_data.py similarity index 100% rename from "day5/\346\274\224\347\277\2223/tests/test_data.py" rename to day5/practice3/tests/test_data.py diff --git "a/day5/\346\274\224\347\277\2223/tests/test_model.py" b/day5/practice3/tests/test_model.py similarity index 79% rename from "day5/\346\274\224\347\277\2223/tests/test_model.py" rename to day5/practice3/tests/test_model.py index e11a19a5c..1314b5ccf 100644 --- "a/day5/\346\274\224\347\277\2223/tests/test_model.py" +++ b/day5/practice3/tests/test_model.py @@ -171,3 +171,35 @@ def test_model_reproducibility(sample_data, preprocessor): assert np.array_equal( predictions1, predictions2 ), "モデルの予測結果に再現性がありません" + + +def test_model_performance_comparison(train_model): + """現在のモデルと過去のモデルの性能を比較""" + model, X_test, y_test = train_model + + # 現在のモデルの精度と推論時間を計算 + current_y_pred = model.predict(X_test) + current_accuracy = accuracy_score(y_test, current_y_pred) + start_time = time.time() + model.predict(X_test) + current_inference_time = time.time() - start_time + + # 過去のモデルをロードして比較 + if os.path.exists(MODEL_PATH): + with open(MODEL_PATH, "rb") as f: + past_model = pickle.load(f) + past_y_pred = past_model.predict(X_test) + past_accuracy = accuracy_score(y_test, past_y_pred) + start_time = time.time() + past_model.predict(X_test) + past_inference_time = time.time() - start_time + + # 精度と推論時間を比較 + assert ( + current_accuracy >= past_accuracy + ), f"現在のモデルの精度が過去のモデルより低いです: {current_accuracy} < {past_accuracy}" + assert ( + current_inference_time <= past_inference_time + ), f"現在のモデルの推論時間が過去のモデルより長いです: {current_inference_time} > {past_inference_time}" + else: + pytest.skip("過去のモデルが存在しないためスキップします") diff --git "a/day5/\346\274\224\347\277\2221/models/titanic_model.pkl" "b/day5/\346\274\224\347\277\2221/models/titanic_model.pkl" deleted file mode 100644 index 6fec87e47..000000000 Binary files "a/day5/\346\274\224\347\277\2221/models/titanic_model.pkl" and /dev/null differ