Build Your Own Bias Measuring and Mitigation Dashboard in 5 Steps

# import libraries

import streamlit as st

import holisticai

import matplotlib.pyplot as plt


from sklearn.model_selection import train_test_split 
from sklearn.metrics import RocCurveDisplay

from sklearn.preprocessing import StandardScaler

from sklearn.linear_model import LogisticRegression

from sklearn.neural_network import MLPClassifier

from sklearn.ensemble import RandomForestClassifier


from holisticai.datasets import load_law_school

from holisticai.bias.plots import group_pie_plot, distribution_plot

from holisticai.bias.mitigation import Reweighing


# set up app title

st.markdown("# Classification with Machine Learning")

# set up sidebar text

st.sidebar.write(

     """This is a dashboard to measure and mitigate bias with

     HolisticAI library.""" 
)


# load dataframe df = load_law_school()['frame']


# create one tab for each step

step1, step2, step3, step4 = st.tabs(["Step 1: Data Description",

                                        "Step 2: Training Model",

                                        "Step 3: Bias Metrics",

                                        "Step 4: Bias Mitigation"])

with step1:


     
    st.subheader("Descriptive Analysis") 


     

    # protected attribute (race) 

    p_attr = df['race1'] 


     

    # binary label vector 

    y = df['bar']         


 

    fig1, ax = plt.subplots(1, 2, figsize=(10,3)) 


     

    # create a pie plot with protected attribute  

    group_pie_plot(p_attr, ax=ax[0])  


 

    # create a distplot with target and gender variables 

    distribution_plot(df['ugpagt3'], df['gender'], ax=ax[1]) 

    plt.tight_layout() 


     

    # show fig1 in app 

    st.pyplot(fig1) 

with step2:



    # load machine learning models

    lr = LogisticRegression()

    rf = RandomForestClassifier()

    mlp = MLPClassifier(hidden_layer_sizes = 10, max_iter=50)


    models = [lr, rf, mlp]


    # model selector

    model = st.selectbox("Select a Model", models)


# simple preprocessing before training.

    df_enc = df.copy()

    df_enc['bar'] = df_enc['bar'].replace({'FALSE':0, 'TRUE':1})


    # split features and target, then train test split

    X = df_enc.drop(columns=['bar', 'ugpagt3'])

    y = df_enc['bar']

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)


    # StandarScaler transformation

    scaler = StandardScaler()

    X_train_t = scaler.fit_transform(X_train.drop(columns=['race1', 'gender']))

    X_test_t = scaler.transform(X_test.drop(columns=['race1', 'gender']))


    # fit model

    model.fit(X_train_t, y_train)


    # predictions

    y_pred = model.predict(X_test_t)


    # create ROC Curve image

    roc_disp = RocCurveDisplay.from_estimator(model, X_test_t, y_test)

    roc_disp.plot()

    st.pyplot() 

with step3:

    # set up groups, prediction array and true (aka target/label) array.

    group_a = X_test["race1"]=='non-white'  # non-white vector

    group_b = X_test["race1"]=='white'      # white vector

    y_true  = y_test                        # true vector


    # create a table with classification bias metrics

    bias_metrics = classification_bias_metrics(group_a, group_b, y_pred, y_test, metric_type='both')


    # show table with bias metrics in app

    st.table(bias_metrics) 

with step4:


    # load mitigator (Reweighing)

    reweighing_mitigator = Reweighing()


    # use mitigator in data

    reweighing_mitigator.fit(y_train, group_a, group_b)


    # access the new sample_weight

    sw = reweighing_mitigator.estimator_params["sample_weight"]


    # training the model

    model.fit(X_train_t, y_train, sample_weight=sw)


    # make predictions

    y_pred = model.predict(X_test_t)


    # create a table with bias metrics for mitigation strategy

    bias_metrics_mitigated = holisticai.bias.metrics.classification_bias_metrics(group_a, group_b, y_pred, y_test, metric_type='both')


    # show table with bias metrics in app

    st.table(bias_metrics_mitigated)