project_cleaning.py

# -*- coding: utf-8 -*-
"""Project_CLEANING.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1bk6_u3x6bQwfmwe1kRjQsBXCjIs1cHuP

# THIS FILE IS WORKING ON INJURIES DATA SET SO FAR...

# STEP 1

github: players.csv (for age), injuries (position )

kaggle: 1st and future(play type,  position), analysis of nfl injuries

**INJURIES DATASETS**

Load csv files

Step 1: combine all of the injury datasets into 1 large data set containing all injuries from 2009-2023
"""

import pandas as pd

from google.colab import files
uploaded=files.upload()

injury2009= pd.read_csv('injuries_2009(1).csv')
injury2010= pd.read_csv('injuries_2010(1).csv')
injury2011= pd.read_csv('injuries_2011.csv')
injury2012= pd.read_csv('injuries_2012.csv')
injury2013= pd.read_csv('injuries_2013.csv')
injury2014= pd.read_csv('injuries_2014.csv')
injury2015= pd.read_csv('injuries_2015.csv')
injury2016= pd.read_csv('injuries_2016.csv')
injury2017= pd.read_csv('injuries_2017.csv')
injury2018= pd.read_csv('injuries_2018.csv')
injury2019= pd.read_csv('injuries_2019.csv')
injury2020= pd.read_csv('injuries_2020.csv')
injury2021= pd.read_csv('injuries_2021.csv')
injury2022= pd.read_csv('injuries_2022.csv')
injury2023= pd.read_csv('injuries_2023.csv')

"""i want to combine all these datasets to have 1 large datasets with all the injuries."""

# Concatenate all datasets
all_injury_data = pd.concat([injury2009, injury2010, injury2011,injury2012,injury2013,injury2014,injury2014,injury2015,injury2016,injury2017,injury2018,injury2019,injury2020,injury2021,injury2022,injury2023], ignore_index=True)

"""Saving this new combined injury dataset to a .csv file"""

all_injury_data.to_csv('all_injury_data.csv', index=False)

"""After doing this, we should quickly analyze our data.


check for null values,return a lsit of all the columns that have null values

"""

for column in all_injury_data.columns:
    null_count = all_injury_data[column].isnull().sum()
    if null_count > 0:
        print(f"Column '{column}' has {null_count} null values.")

"""Lets analyze this information.

We have 2 kinds of injuries here- injury in games or practice.


* report primary injury having null values makes sense, as this means the player did not get injured during the game, but perhaps theres a value for injury in practice. it could be assumed that if the player wasnt injured in game, they would have been injred in practice ( otherwise they wouldnt be in this table because theyd have no injuries)

* secondary injury having null values makes sense, because its not always the case that a player will receive a secondary injury. sometimes its only a primary injury. We would assume that if secondary injury is empty, then primary injury should also be empty...


* Practice primary injury having null values makes sense, as its possible a player was injured during the game and not practice. it could be assumed that if the player wasnt injured in practice, they would have been injred in game ( otherwise they wouldnt be in this table because theyd have no injuries)


* practice secondary injury having null values makes sense,because its not always the case that a player will receive a secondary injury. sometimes its only a primary injury. We would assume that if secondary injury is empty, then primary injury should also be empty...


* Date modified having null values may or may not make sense, but in regards to our task we dont care about this value. perhaps we can drop this column

* Report status having null values makes sense. As we see , the number of missing report status is the same as the number of missing primary injury. We could assume that these missing report statuses are because the player was injured during practice not in game. (to be confirmed later)

After analyzing our data, we see that our null values are not necessarily problem values, they make sense in this case. Despite this, null vlaues have to be cleaned in order to prepare for the model . We will move onto this later.

**Checking for Contradicting data/ noisy data**
"""

# Check if there are any entries where both primary injury and practice primary injury are both null
#This would be a noisy data, because there should not be any values without either of tehse injuries because it would mean
#the player was not injured
missing_injuries = all_injury_data[(all_injury_data['report_primary_injury'].isnull()) &
                                   (all_injury_data['practice_primary_injury'].isnull())]
# Print the length of missing_injuries
print("Length of missing_injuries:", len(missing_injuries))

# Check if there are any entries
if missing_injuries.empty:
    print("None")
else:
    # Display the resulting DataFrame
    print(missing_injuries)

# Checking if theres any case where a player has a secondary injury but not a primary injury, this would be noisy data
secondary_injury_condition = ((~all_injury_data['report_secondary_injury'].isnull()) & all_injury_data['report_primary_injury'].isnull() | (~all_injury_data['practice_secondary_injury'].isnull()) & all_injury_data['practice_primary_injury'].isnull())

# Filter the dataset based on the condition
inconsistent_secondary_injuries = all_injury_data[secondary_injury_condition]

# Check if there are any entries
if inconsistent_secondary_injuries.empty:
    print("None")
    # Print the length of inconsistent_secondary_injuries
    print("Length of inconsistent_secondary_injuries:", len(inconsistent_secondary_injuries))
else:
    # Display the resulting DataFrame
    print(inconsistent_secondary_injuries)
    # Print the length of inconsistent_secondary_injuries
    print("Length of inconsistent_secondary_injuries:", len(inconsistent_secondary_injuries))

"""**Beginning the cleaning process for the injuries datasets**

As mentioned, we dont really need the date_modified column. it has no value to our task and has null values. so we can remove it
"""

all_injury_data2 = all_injury_data.drop('date_modified', axis=1, errors='ignore')

# Check the updated dataset
print(all_injury_data2.head())

"""Now, we are going to drop the  practice injuries, as we are only concerned with injuries in game."""

all_injury_data2 = all_injury_data2.drop('practice_primary_injury', axis=1, errors='ignore')

all_injury_data2 = all_injury_data2.drop('practice_secondary_injury', axis=1, errors='ignore')

all_injury_data2 = all_injury_data2.drop('practice_status', axis=1, errors='ignore')

"""We are going to drop secondary injuries, for the scope of this project we are mostly interested in primary injuries directly caused by the game."""

all_injury_data2 = all_injury_data2.drop('report_secondary_injury', axis=1, errors='ignore')
# Check the updated dataset
print(all_injury_data2.head())

"""Now i want to analyze the types of injuries this file holds. im going to return all the different injuries"""

report_primary_injuries = all_injury_data2['report_primary_injury']

# Print all values in the 'report_primary_injury' column
print(report_primary_injuries)

"""we see there are null vlaues, its possible that these are null because in the original DF these entities got practice injuries and not game injuries. lets check our suspicions"""

# Create a DataFrame with entries having null 'report_primary_injury'
null_report_primary_entries = all_injury_data[all_injury_data['report_primary_injury'].isnull()]

# Initialize a counter
count_null_report_with_practice_primary = 0

# Loop through the DataFrame and check for corresponding 'practice_primary_injury'
for index, row in null_report_primary_entries.iterrows():
    if pd.notnull(row['practice_primary_injury']):
        count_null_report_with_practice_primary += 1

# Print the count
print(f"Count of entries with null report_primary_injury and non-null practice_primary_injury: {count_null_report_with_practice_primary}")
print("Total Null Values in report_primary_injury 22352")

"""Our assumption was right, all of the null values in this column are due to the fact that the player was not injured in game, but was injured during a practice, therefore we can remove these entities from the dataset as they are not valuable to us."""

# Remove rows with null 'report_primary_injury'
all_injury_data2 = all_injury_data2.dropna(subset=['report_primary_injury'])

# Print the total number of entities after removal
total_entities_after_removal = len(all_injury_data2)
print(f"Total number of entities after removing null values in 'report_primary_injury': {total_entities_after_removal}")

print(all_injury_data2)

def count_null_and_filter(df):
    # Count null values in 'report_status' column
    null_count = df['report_status'].isnull().sum()

    # Filter rows where 'report_status' is null
    null_report_status_df = df[df['report_status'].isnull()][['report_status', 'report_primary_injury']]

    return null_count, null_report_status_df

# Call the function with your DataFrame
null_count, null_report_status_df = count_null_and_filter(all_injury_data2)

# Print the results
print("Count of null values in 'report_status':", null_count)
print("\nDataFrame with null 'r'")

"""As we can see, by cleaning the report primary injury column, we have subsequently also cleaned the report_status column. Our initial assumption was right. Our dataset should now be cleaned from null values.

**Analyzing and Cleaning Injury column**

We want to analyze the injuries column. Below we report a list of all unique injury values.

We will first convert all values to lowercase, this way Knee and knee will not be considered 2 unique values
"""

#first we convert everything to lower case
unique_primary_injuries = all_injury_data2['report_primary_injury'].str.lower().unique()

# Convert the result to a list
unique_primary_injuries_list = list(unique_primary_injuries)

# Print the list of unique injuries
print(unique_primary_injuries_list)

# Print the count of items in the list
print("Count of unique primary injuries:", len(unique_primary_injuries_list))

"""There are 182 unique primary injury values"""

primary_injury_counts = all_injury_data2['report_primary_injury'].str.lower().value_counts()

# Convert the result to a DataFrame for clarity
primary_injury_counts_df = pd.DataFrame({'Injury Type': primary_injury_counts.index, 'Count': primary_injury_counts.values})

# Print the DataFrame with unique values and their counts
print(primary_injury_counts_df)

"""As we can see , there are 182 unique values for injury.

There are major issues with the way the injuries are being identified. for example, there are columns like

*   finger, then right finger
*   elbow, then right elbow
*   r shoulder, then right shoulder

There are multiple symbols representing right and left


*   right, rt , r.
*   left, lf, l

we need to clean the injury column by doing entity resolution.

We should identify which entities mean the same thing, and replace their value with a common value

We also notice there are some nan values

**Entity Resolution for Injury Types**

Part 1: Identify Potential Similar Entities using  Similarity methods
"""

all_injury_data2.to_csv('all_injury_data2.csv', index=False)

!pip install fuzzywuzzy

from fuzzywuzzy import fuzz

# Assuming unique_combined_list contains your data
#i am only running the similarity on the unique values, because we know values like knee and knee will be
#100% unique, and those dont concern us for now
threshold = 80  # Adjust as needed

# Use fuzzy matching to find similar strings
similar_strings = []
for i in range(len(unique_primary_injuries_list)):
    for j in range(i+1, len(unique_primary_injuries_list)):
        similarity = fuzz.token_sort_ratio(unique_primary_injuries_list[i], unique_primary_injuries_list[j])
        if similarity >= threshold:
            similar_strings.append((unique_primary_injuries_list[i], unique_primary_injuries_list[j]))

# Print similar strings
for pair in similar_strings:
    print(f"Similar: {pair[0]} ---- {pair[1]}")

# Count of similar pairs
print(f"\nCount of similar pairs: {len(similar_strings)}")

"""There are 28 similar pairs that are potentially the same entity

1.   Things with S are considered the same entity- (lacerations and laceration)
2.  Things with a left/right prefix are generalized to just the item
3. Spelling errors (qblique/oblique)
4. Things with brackets or slashes (migraines)vs migraines. foot vs /foot

STEP 1: Changing entities with 's' in them

Similar: hamstring ---- hamstrings


Similar: ribs ---- rib

Similar: knee ---- knees

Similar: shoulder ---- shoulders

Similar: quadricep ---- quadriceps

Similar: ankle ---- ankles

Similar: hand ---- hands

Similar: toe ---- toes

Similar: hip ---- hips

Similar: biceps ---- bicep

Similar: triceps ---- tricep

Similar: migraines ---- migraine

Similar: facial lacerations ---- facial laceration

Similar: covid protocol ---- covid protocols
"""

# Step 1: Convert all values in 'report_primary_injury' to lowercase
all_injury_data2['report_primary_injury'] = all_injury_data2['report_primary_injury'].str.lower()

# Step 2: Filter values that end with the letter 's'
values_ending_with_s = all_injury_data2[all_injury_data2['report_primary_injury'].str.endswith('s')]['report_primary_injury']

# Step 3: Get unique values from the filtered DataFrame
unique_values_ending_with_s = values_ending_with_s.unique()

# Print the result
print("Values ending with 's' in 'report_primary_injury' (unique):")
print(unique_values_ending_with_s)

# Define a custom function to replace values
def replace_values(value):
    exceptions = ["illness", "back spasms", "achilles","pelvis", "chest/ankle/illness", "buttocks",
                  "solar plexus", "ribs/shoulder/illness", "trapezius", "medical illness",
                  "non-football illness", "knee illness", "appendicitis"]

    if value in exceptions:
        return value  # Do not modify exceptions
    elif value.endswith('s'):
        return value[:-1]  # Remove 's' from the end of the string
    else:
        return value

# Apply the custom function to the 'report_primary_injury' column
all_injury_data2['report_primary_injury'] = all_injury_data2['report_primary_injury'].apply(replace_values)

unique_primary_injuries = all_injury_data2['report_primary_injury'].unique()

# Convert the result to a list
unique_primary_injuries_list = list(unique_primary_injuries)

# Print the list of unique injuries
print(unique_primary_injuries_list)

# Print the count of items in the list
print("Count of unique primary injuries:", len(unique_primary_injuries_list))

"""We have lowered the number of unique injury values from 182 to 168.

Step 2: Things with left/right as a prefix are generalized
"""

threshold = 80  # Adjust as needed

# Use fuzzy matching to find similar strings
similar_strings = []
for i in range(len(unique_primary_injuries_list)):
    for j in range(i+1, len(unique_primary_injuries_list)):
        similarity = fuzz.token_sort_ratio(unique_primary_injuries_list[i], unique_primary_injuries_list[j])
        if similarity >= threshold:
            similar_strings.append((unique_primary_injuries_list[i], unique_primary_injuries_list[j]))

# Print similar strings
for pair in similar_strings:
    print(f"Similar: {pair[0]} ---- {pair[1]}")

# Count of similar pairs
print(f"\nCount of similar injuries: {len(similar_strings)}")

"""count of similar pairs are now 13 nstead of 28"""

# Define a custom function to remove prefixes
def remove_prefix(value):
    prefixes = ["right ", "left ", "rt. ", "lf. ", "r ", "l."]

    for prefix in prefixes:
        if value.lower().startswith(prefix):
            return value[len(prefix):]  # Remove the prefix
    return value

# Apply the custom function to the 'report_primary_injury' column
all_injury_data2['report_primary_injury'] = all_injury_data2['report_primary_injury'].apply(remove_prefix)

# Print the modified column
print("Modified 'report_primary_injury' column:")
print(all_injury_data2['report_primary_injury'])

unique_primary_injuries = all_injury_data2['report_primary_injury'].unique()

# Convert the result to a list
unique_primary_injuries_list = list(unique_primary_injuries)

# Print the count of items in the list
print("Count of unique primary injuries:", len(unique_primary_injuries_list))

"""Now we have lowered the unique values for injury from 168 to 130"""

threshold = 80  # Adjust as needed

# Use fuzzy matching to find similar strings
similar_strings = []
for i in range(len(unique_primary_injuries_list)):
    for j in range(i+1, len(unique_primary_injuries_list)):
        similarity = fuzz.token_sort_ratio(unique_primary_injuries_list[i], unique_primary_injuries_list[j])
        if similarity >= threshold:
            similar_strings.append((unique_primary_injuries_list[i], unique_primary_injuries_list[j]))

# Print similar strings
for pair in similar_strings:
    print(f"Similar: {pair[0]} ---- {pair[1]}")

# Count of similar pairs
print(f"\nCount of similar injuries: {len(similar_strings)}")

"""count of similar pairs is now 8

STEP 3: things with brackets
"""

import re

# Define a custom function to remove strings wrapped in brackets
def remove_brackets(value):
    # Use regular expression to find strings in brackets and remove them
    return re.sub(r'\((.*?)\)', '', value)

# Apply the custom function to the 'report_primary_injury' column
all_injury_data2['report_primary_injury'] = all_injury_data2['report_primary_injury'].apply(remove_brackets)

# Print the modified column
print("Modified 'report_primary_injury' column:")
print(all_injury_data2['report_primary_injury'])

unique_primary_injuries = all_injury_data2['report_primary_injury'].unique()

# Convert the result to a list
unique_primary_injuries_list = list(unique_primary_injuries)

# Print the count of items in the list
print("Count of unique primary injuries:", len(unique_primary_injuries_list))

threshold = 80  # Adjust as needed

# Use fuzzy matching to find similar strings
similar_strings = []
for i in range(len(unique_primary_injuries_list)):
    for j in range(i+1, len(unique_primary_injuries_list)):
        similarity = fuzz.token_sort_ratio(unique_primary_injuries_list[i], unique_primary_injuries_list[j])
        if similarity >= threshold:
            similar_strings.append((unique_primary_injuries_list[i], unique_primary_injuries_list[j]))

# Print similar strings
for pair in similar_strings:
    print(f"Similar: {pair[0]} ---- {pair[1]}")

# Count of similar pairs
print(f"\nCount of similar injuries: {len(similar_strings)}")

"""count of similar injuries now 6

STEP 4: typos
"""

# Define a custom function to replace values
def replace_qblique(value):
  value.replace('qblique', 'oblique')
  value = value.replace('oblique ', 'oblique')
  return value.replace(' oblique', 'oblique')

# Apply the custom function to the 'report_primary_injury' column
all_injury_data2['report_primary_injury'] = all_injury_data2['report_primary_injury'].apply(replace_qblique)

# Print the modified column
print("Modified 'report_primary_injury' column:")
print(all_injury_data2['report_primary_injury'])

unique_primary_injuries = all_injury_data2['report_primary_injury'].unique()

# Convert the result to a list
unique_primary_injuries_list = list(unique_primary_injuries)

# Print the count of items in the list
print("Count of unique primary injuries:", len(unique_primary_injuries_list))



"""count of similar injuries now 129"""

threshold = 80  # Adjust as needed

# Use fuzzy matching to find similar strings
similar_strings = []
for i in range(len(unique_primary_injuries_list)):
    for j in range(i+1, len(unique_primary_injuries_list)):
        similarity = fuzz.token_sort_ratio(unique_primary_injuries_list[i], unique_primary_injuries_list[j])
        if similarity >= threshold:
            similar_strings.append((unique_primary_injuries_list[i], unique_primary_injuries_list[j]))

# Print similar strings
for pair in similar_strings:
    print(f"Similar: {pair[0]} ---- {pair[1]}")

# Count of similar pairs
print(f"\nCount of similar pairs: {len(similar_strings)}")

"""count of similar pairs is now 6

STEP 5: custom cases

changing values on a case by case basis
"""

def replace_values(value):
    value = value.replace('both knee', 'knee')
    value = value.replace('appendix', 'appendicitis')
    value= value.replace('teeth', 'tooth')
    value= value.replace('low back', 'lower back')
    value= value.replace('illness ', 'illness')
    value= value.replace(' arm', 'arm')
    value= value.replace('qblique', 'oblique')
    return value

# Apply the custom function to the 'report_primary_injury' column
all_injury_data2['report_primary_injury'] = all_injury_data2['report_primary_injury'].apply(replace_values)

unique_primary_injuries = all_injury_data2['report_primary_injury'].unique()

# Convert the result to a list
unique_primary_injuries_list = list(unique_primary_injuries)

# Print the count of items in the list
print("Count of unique primary injuries:", len(unique_primary_injuries_list))

"""count of unique primary injuries is now at 124"""

print(unique_primary_injuries_list)
print(len(unique_primary_injuries_list))

"""The bulk of the entity resolution is done. But now we want to generalize some more injuries. For the sake and scope of our project, we believe more general injuries would  be preferable, so we will change things like



*   tailbone, buttocks, glute  ---> glute
*   lower back --> back
*  lung contusion --> lung
*  upper arm / lower arm --> arm
* toe, heel --> foot
* finger, thumb --> hand
* knee illness --> knee
* reserve/covid activation, covid/reserve, covid protocol --> covid
* core muscle injury , abdominal, core muscle --> core
* facial laceration , face, nose, cheek, eyelid, jaw , ear, eye  --> face
* lower leg --> leg



"""

def replace_values(value):
    value = value.replace('tailbone', 'glute')
    value = value.replace('buttocks', 'glute')
    value = value.replace('lower back', 'back')
    value = value.replace('lumbar', 'back')
    value = value.replace('back spasms', 'back')
    value = value.replace('lung contusion', 'lung')
    value = value.replace('upper arm', 'arm')
    value = value.replace('upperarm', 'arm')
    value = value.replace('toe', 'foot')
    value = value.replace('heel', 'foot')
    value = value.replace('feet', 'foot')
    value = value.replace('finger', 'hand')
    value = value.replace('thumb', 'hand')
    value = value.replace('knee illness', 'knee')
    value = value.replace('reserve/covid activation', 'covid')
    value = value.replace('covid/reserve', 'covid')
    value = value.replace('covid protocol', 'covid')
    value = value.replace('covid ramp up', 'covid')
    value = value.replace('core muscle injury', 'core')
    value = value.replace('abdominal', 'core')
    value = value.replace('abdomen', 'core')
    value = value.replace('core muscle', 'core')
    value = value.replace('facial laceration', 'face')
    value = value.replace('nose', 'face')
    value = value.replace('cheek', 'face')
    value = value.replace('mouth', 'face')
    value = value.replace('eyelid', 'face')
    value = value.replace('jaw', 'face')
    value = value.replace('ear', 'face')
    value = value.replace('eye', 'face')
    value = value.replace('chin', 'face')
    value = value.replace('forfacem', 'face')
    value = value.replace('face lid', 'face')
    value = value.replace('lower leg', 'leg')
    value = value.replace('fibula', 'calf')
    value = value.replace('rib cage', 'rib')
    value = value.replace('hip flexor', 'hip')
    value = value.replace('flu', 'illness')
    value = value.replace('lacerated kidney', 'kidney')
    value = value.replace('ribs ', 'rib')
    value = value.replace('rib ', 'rib')
    value = value.replace('medical illness', 'illness')
    value = value.replace('appendicitis', 'illness')
    value = value.replace('infection', 'illness')
    value = value.replace('arrhythmia', 'illness')
    value = value.replace('cardiac', 'illness')
    value = value.replace('lung', 'illness')
    value = value.replace('shortness of breath', 'illness')
    value = value.replace('trapezius', 'illness')
    value = value.replace('solar plexus', 'illness')
    value = value.replace('heat cramp', 'illness')
    value = value.replace('arch', 'feet')
    value = value.replace('hernia', 'illness')
    value = value.replace('migraine', 'illness')
    value = value.replace('covid', 'illness')
    value = value.replace('spleen', 'illness')
    value = value.replace('liver', 'illness')
    value = value.replace('kidney', 'illness')
    value = value.replace('stomach', 'illness')
    value = value.replace('sternoclavicular', 'collarbone')
    value = value.replace('tibia', 'shin')
    value = value.replace('quadricep', 'quad')
    value = value.replace('adductor', 'hip')
    value = value.replace('pelvis', 'groin')
    value = value.replace('other-stinger', 'stinger')
    value = value.replace('tooth', 'face')
    return value

# Apply the custom function to the 'report_primary_injury' column
all_injury_data2['report_primary_injury'] = all_injury_data2['report_primary_injury'].apply(replace_values)

unique_primary_injuries = all_injury_data2['report_primary_injury'].unique()

# Convert the result to a list
unique_primary_injuries_list = list(unique_primary_injuries)

# Print the count of items in the list
print("Count of unique primary injuries:", len(unique_primary_injuries_list))

"""**Cleaning entries with multiple injuries, example chest/foot**

# SPLITTING / AND , INTO SEPARATE ENTRIES
"""

# Split values in the 'report_primary_injury' column by '/'
all_injury_data2['report_primary_injury'] = all_injury_data2['report_primary_injury'].str.split('/')

# Explode the lists in the 'report_primary_injury' column to separate rows
all_injury_data2 = all_injury_data2.explode('report_primary_injury')

# Reset the index after exploding the DataFrame
all_injury_data2 = all_injury_data2.reset_index(drop=True)

# Print the modified DataFrame
print(all_injury_data2)

# Split values in the 'report_primary_injury' column by ','
all_injury_data2['report_primary_injury'] = all_injury_data2['report_primary_injury'].str.split(',')

# Explode the lists in the 'report_primary_injury' column to separate rows
all_injury_data2 = all_injury_data2.explode('report_primary_injury')

# Reset the index after exploding the DataFrame
all_injury_data2 = all_injury_data2.reset_index(drop=True)

# Print the modified DataFrame
print(all_injury_data2)

unique_primary_injuries = all_injury_data2['report_primary_injury'].unique()

# Convert the result to a list
unique_primary_injuries_list = list(unique_primary_injuries)

# Print the count of items in the list
print("Count of unique primary injuries:", len(unique_primary_injuries_list))

# Loop through the unique primary injuries and count occurrences
for injury in unique_primary_injuries_list:
    count = all_injury_data2['report_primary_injury'].str.lower().value_counts().get(injury.lower(), 0)
    print(f"{injury}: {count}")

"""Since our goal is to predict injury risk of football players, we can safely remove the entities whos primary_injury_value is not injury related, or non football illness or non football injury"""

# Values to remove
invalid_values = ['injury related', 'non football illness', 'non football injury', 'not injury related' , 'non-football illness', 'not injury related - personal matter'
, 'not injury related - discipline','not injury related - resting player', 'not injury related - did not travel','not injury related - other','not injury related - team decision',
                  'not injury related - returning from suspension']

# Filter rows to remove based on invalid values
#making a new dataframe from here on out
all_injury_data3 = all_injury_data2[~all_injury_data2['report_primary_injury'].str.lower().isin(invalid_values)]

all_injury_data3.to_csv('all_injury_data3.csv', index=False)

unique_primary_injuries = all_injury_data3['report_primary_injury'].unique()

# Convert the result to a list
unique_primary_injuries_list = list(unique_primary_injuries)

# Print the count of items in the list
print("Count of unique primary injuries:", len(unique_primary_injuries_list))

# Loop through the unique primary injuries and count occurrences
for injury in unique_primary_injuries_list:
    count = all_injury_data3['report_primary_injury'].str.lower().value_counts().get(injury.lower(), 0)
    print(f"{injury}: {count}")

"""Removing empty strings"""

import numpy as np

# Drop rows with an empty string or where the length of 'report_primary_injury' is <= 1
all_injury_data3 = all_injury_data3[all_injury_data3['report_primary_injury'].str.len() > 1]

"""Now removing any preciding spaces before each injury"""

all_injury_data3['report_primary_injury'] = all_injury_data3['report_primary_injury'].str.strip()

# Drop rows with a count of zero for 'report_primary_injury'
all_injury_data3 = all_injury_data3[all_injury_data3.groupby('report_primary_injury')['report_primary_injury'].transform('count') > 0]

"""Changing the value of ribs to rib"""

# Replace "ribs" with "rib" in 'report_primary_injury'
all_injury_data3['report_primary_injury'] = all_injury_data3['report_primary_injury'].replace('ribs', 'rib')

"""More cleaning"""

# Replace specified values in 'report_primary_injury'
replace_values = {
    'feet': 'foot',
    'throat': 'neck',
    'pec': 'chest',
    'left elbow': 'elbow'
}
all_injury_data3['report_primary_injury'] = all_injury_data3['report_primary_injury'].replace(replace_values)

# Remove values containing "illnes"
all_injury_data3 = all_injury_data3[~all_injury_data3['report_primary_injury'].str.lower().str.contains('illnes')]

# Get unique primary injuries list
unique_primary_injuries_list = all_injury_data3['report_primary_injury'].str.lower().unique()

# Loop through the unique primary injuries and count occurrences
for injury in unique_primary_injuries_list:
    count = all_injury_data3['report_primary_injury'].str.lower().value_counts().get(injury.lower(), 0)
    if count > 0:
        print(f"{injury}: {count}")

"""**mAPPING INJURY VALUES TO A DICTIONARY**

idk if we should do this rn, or only when we start modeling, but basically im pretty sure any non-numeric string when u start modeling has to be converted to numbers cuz the models dont understand strings. so eventually we have to map these for example

hamstring=0 rib=1  back=2...
"""

all_injury_data3.to_csv('all_injury_data_new.csv', index=False)

print(all_injury_data3)