Fairness Test: XGB2

Experiment initialization and data preparation

from piml import Experiment
from piml.models import XGB2Classifier

exp = Experiment()
exp.data_loader("SimuCredit", silent=True)
exp.data_summary(feature_exclude=["Race", "Gender"], silent=True)
exp.data_prepare(target="Approved", task_type="classification", silent=True)

Train Model

exp.model_train(XGB2Classifier(max_depth=2,
                               n_estimators=100,
                               mono_increasing_list=["Mortgage", "Balance"],
                               mono_decreasing_list=["Amount Past Due", "Utilization", "Delinquency",
                                                     "Credit Inquiry", "Open Trade"]),
                name="XGB2_monotonic")

Fairness Metric

metrics_result = exp.model_fairness(model="XGB2_monotonic",
                                    show="metrics",
                                    metric="AIR",
                                    group_category=["Race", "Gender"],
                                    reference_group=[1., 1.],
                                    protected_group=[0., 0.],
                                    favorable_threshold=0.5,
                                    performance_metric="ACC",
                                    return_data=True,
                                    figsize=(5, 4))
metrics_result.data

	Group Index	Group Category	Reference Group	Protected Group	AIR
0	0	Race	1.0	0.0	0.612
1	1	Gender	1.0	0.0	0.745

Fairness Segmented

segmented_result = exp.model_fairness(model="XGB2_monotonic",
                                      show="segmented",
                                      metric="AIR",
                                      segment_feature="Balance",
                                      group_category=["Race", "Gender"],
                                      reference_group=[1., 1.],
                                      protected_group=[0., 0.],
                                      favorable_threshold=0.5,
                                      segment_bins=5,
                                      return_data=True,
                                      figsize=(10, 4))
segmented_result.data

	Segment	Lower Bound	Upper Bound	0-Race | AIR	1-Gender | AIR
0	0	0.97	306.61	0.4485	0.6607
1	1	306.62	601.40	0.4767	0.6426
2	2	601.47	1027.23	0.6319	0.6556
3	3	1027.29	1864.90	0.6486	0.7978
4	4	1864.94	20384.87	0.8781	0.8266

Fairness Binning

binning_result = exp.model_fairness(model="XGB2_monotonic", show="binning",
                                    metric="AIR",
                                    group_category=["Race", "Gender"],
                                    reference_group=[1., 1.],
                                    protected_group=[0., 0.],
                                    favorable_threshold=0.5,
                                    performance_metric="F1",
                                    binning_dict={"Balance": {"type": "quantile", "value": [1, 5]},
                                                  "Mortgage": {"type": "uniform", "value": [1, 5]},
                                                  "Amount Past Due": {"type": "custom", "value": (0, 100)}},
                                    return_data=True,
                                    figsize=(10, 4))
binning_result.data

	ID	Configuration	Gender | AIR	Race | AIR	F1
0	0	Original	0.7450	0.6120	0.7163
1	1	Balance: 1, Mortgage: 1, Amount Past Due: (0, 100)	1.0004	1.0063	0.5049
2	2	Balance: 1, Mortgage: 2, Amount Past Due: (0, 100)	1.0009	1.0008	0.5064
3	3	Balance: 1, Mortgage: 3, Amount Past Due: (0, 100)	1.0032	0.9776	0.5127
4	4	Balance: 1, Mortgage: 4, Amount Past Due: (0, 100)	1.0048	0.9273	0.5276
5	5	Balance: 1, Mortgage: 5, Amount Past Due: (0, 100)	1.0160	0.8439	0.5374
6	6	Balance: 2, Mortgage: 1, Amount Past Due: (0, 100)	0.5674	0.9873	0.3577
7	7	Balance: 2, Mortgage: 2, Amount Past Due: (0, 100)	0.5718	0.9743	0.3611
8	8	Balance: 2, Mortgage: 3, Amount Past Due: (0, 100)	0.5805	0.9321	0.3713
9	9	Balance: 2, Mortgage: 4, Amount Past Due: (0, 100)	0.5921	0.8310	0.3991
10	10	Balance: 2, Mortgage: 5, Amount Past Due: (0, 100)	0.5918	0.6627	0.4014
11	11	Balance: 3, Mortgage: 1, Amount Past Due: (0, 100)	0.4346	1.0061	0.3877
12	12	Balance: 3, Mortgage: 2, Amount Past Due: (0, 100)	0.4388	0.9928	0.3915
13	13	Balance: 3, Mortgage: 3, Amount Past Due: (0, 100)	0.4569	0.9530	0.4011
14	14	Balance: 3, Mortgage: 4, Amount Past Due: (0, 100)	0.4852	0.8571	0.4290
15	15	Balance: 3, Mortgage: 5, Amount Past Due: (0, 100)	0.5129	0.7078	0.4340
16	16	Balance: 4, Mortgage: 1, Amount Past Due: (0, 100)	0.4720	0.9962	0.4153
17	17	Balance: 4, Mortgage: 2, Amount Past Due: (0, 100)	0.4746	0.9865	0.4179
18	18	Balance: 4, Mortgage: 3, Amount Past Due: (0, 100)	0.4921	0.9476	0.4276
19	19	Balance: 4, Mortgage: 4, Amount Past Due: (0, 100)	0.5191	0.8632	0.4516
20	20	Balance: 4, Mortgage: 5, Amount Past Due: (0, 100)	0.5259	0.6781	0.4292
21	21	Balance: 5, Mortgage: 1, Amount Past Due: (0, 100)	0.4835	1.0036	0.4054
22	22	Balance: 5, Mortgage: 2, Amount Past Due: (0, 100)	0.4873	0.9920	0.4087
23	23	Balance: 5, Mortgage: 3, Amount Past Due: (0, 100)	0.5048	0.9508	0.4186
24	24	Balance: 5, Mortgage: 4, Amount Past Due: (0, 100)	0.5227	0.8739	0.4412
25	25	Balance: 5, Mortgage: 5, Amount Past Due: (0, 100)	0.5172	0.7055	0.4302

Fairness Thresholding

thresholding_result = exp.model_fairness(model="XGB2_monotonic",
                                         show="thresholding",
                                         metric="AIR",
                                         group_category=["Race", "Gender"],
                                         reference_group=[1., 1.],
                                         protected_group=[0., 0.],
                                         favorable_threshold=0.32,
                                         performance_metric="ACC",
                                         return_data=True,
                                         figsize=(10, 4))
thresholding_result.data

	Group Index	Group Category	Reference Group	Protected Group	Threshold	AIR	ACC
0	0	Race	1.0	0.0	0.3324	0.8862	0.6757
1	1	Gender	1.0	0.0	0.3324	0.8917	0.6757

Estimated memory usage: 23 MB