preprocess_dataset.py

"""Clean and normalize the raw student dataset.

Fixes applied to the raw export (data/raw/students_raw.csv):
1. Participation_Score was on a 0-10 scale while every other score column was on
   a 0-100 scale. It is normalized to 0-100 (value * 10).
2. Total_Score was not derived from the component scores (correlation ~0). It is
   now computed from a documented weighted formula:
       Total = 0.20*Midterm + 0.25*Final + 0.20*Assignments
             + 0.15*Quizzes + 0.10*Projects + 0.10*Participation
3. Grade was randomly assigned (only ~21% of rows were consistent with the
   total score). It is now derived from Total_Score using the standard
   Indonesian university scale:
       A >= 80, B >= 70, C >= 60, D >= 50, F < 50

Run from the project root:
    python scripts/preprocess_dataset.py
"""

from pathlib import Path

import pandas as pd

RAW_PATH = Path(__file__).resolve().parent.parent / "data" / "raw" / "students_raw.csv"
OUT_PATH = Path(__file__).resolve().parent.parent / "data" / "students.csv"

GRADE_BINS = [0, 50, 60, 70, 80, 101]
GRADE_LABELS = ["F", "D", "C", "B", "A"]

WEIGHTS = {
    "Midterm_Score": 0.20,
    "Final_Score": 0.25,
    "Assignments_Avg": 0.20,
    "Quizzes_Avg": 0.15,
    "Projects_Score": 0.10,
    "Participation_Score": 0.10,
}


def main() -> None:
    if not RAW_PATH.exists():
        raise FileNotFoundError(f"Raw dataset not found: {RAW_PATH}")

    df = pd.read_csv(RAW_PATH)
    original_rows = len(df)

    df["Participation_Score"] = (df["Participation_Score"] * 10).round(2)

    df["Total_Score"] = sum(
        df[col] * weight for col, weight in WEIGHTS.items()
    ).round(2)

    df["Grade"] = pd.cut(
        df["Total_Score"], bins=GRADE_BINS, labels=GRADE_LABELS, right=False
    ).astype(str)

    OUT_PATH.parent.mkdir(parents=True, exist_ok=True)
    df.to_csv(OUT_PATH, index=False)

    print(f"Rows processed : {original_rows} -> {len(df)}")
    print(f"Output saved   : {OUT_PATH}")
    print()
    print("Grade distribution:")
    for grade in GRADE_LABELS:
        count = int((df["Grade"] == grade).sum())
        mean_total = df.loc[df["Grade"] == grade, "Total_Score"].mean()
        print(f"  {grade}: {count} students, Total_Score mean = {mean_total:.2f}")
    print()
    print("Checks:")
    print(f"  Participation in 0-100 : {bool(df['Participation_Score'].between(0, 100).all())}")
    print(f"  Total_Score in 0-100   : {bool(df['Total_Score'].between(0, 100).all())}")
    print(f"  No NaN / duplicates    : {bool(df.isna().sum().sum() == 0 and not df.duplicated().any())}")


if __name__ == "__main__":
    main()