Mitigating Projections

In this video we will try to see if we can mitigate the bias in word embeddings by using a linear algebra trick.

Video#

Code#

Here's the code from the previous video.

from whatlies import Embedding, EmbeddingSet
from whatlies.transformers import Pca
from whatlies.language import FasttextLanguage, SpacyLanguage, BytePairLang

lang_ft = FasttextLanguage("cc.en.300.bin")
lang_sp = SpacyLanguage("en_core_web_sm")

stereotype_pairs = [
    ('sewing', 'carpentry'),
    ('nurse', 'physician'),
    ('nurse', 'surgeon'),
    ('nurse', 'doctor'),
    ('diva', 'rockstar'),
]

appropriate_pairs = [
    ('woman', 'man'),
    ('queen', 'king'),
    ('sister', 'brother'),
    ('mother', 'father'),
    ('ovarian', 'prostate'),
    ('she', 'he'),
    ('her', 'him'),
    ('girl', 'boy')
]

random_pairs = [
    ('dog', 'firehydrant'),
    ('cat', 'record'),
    ('carpet', 'leg'),
    ('hot', 'cold'),
    ('coffee', 'milk'),
    ('fire', 'ice'),
    ('rich', 'poor'),
    ('more', 'less'),
    ('math', 'language')
]

flatten = lambda l: [item for sublist in l for item in sublist]

def calc_axis(pair_list, language_model):
    return [language_model[t1] - language_model[t2] for (t1, t2) in pair_list]

def make_correlation_plot(pairs, language_model, metric="cosine"):
    emb_pairs = EmbeddingSet(*flatten([calc_axis(p, language_model) for p in pairs]))
    emb_pairs.plot_distance(metric=metric)

make_correlation_plot(pairs=[stereotype_pairs, appropriate_pairs, random_pairs], language_model=lang_ft)

We will now generate a similar chart but now with the projection on the "he-she"-direction.

def calc_axis(pair_list, language_model):
    return [language_model[t1] - language_model[t2] for (t1, t2) in pair_list]

def make_debias_correlation_plot(pairs, language_model, metric='cosine'):
    emb_pairs = EmbeddingSet(*flatten([calc_axis(p, language_model) for p in pairs]))
    # Generate an average vector on the "gender" direction.
    norm_emb = EmbeddingSet(
        (language_model['man'] - language_model['woman']),
        (language_model['king'] - language_model['queen']),
        (language_model['father'] - language_model['mother'])
    ).average()
    # Project the embeddingset away
    emb_pairs = emb_pairs | norm_emb
    emb_pairs.plot_distance(metric=metric)
    return emb_pairs

_ = make_debias_correlation_plot(pairs=[stereotype_pairs, appropriate_pairs, random_pairs],
                                 language_model=BytePairLang("en", dim=300, vs=200_000))

Exercises#

Try to answer the following questions to test your knowledge.

1. We'll see in a later video that this project tricks comes with some flaws. Can you predict where the flaws are?
2. If you feel like doing a programming exercise; try to repeat the exercise but with a different kind of bias.