Vectorization in NLP

Vectorization in NLP:

Vectorization is the process of converting text into numbers.

Techniques for Vectorization:

• Bag of Words (BoW)
• TF-IDF

Implementation of BoW:

1. Using Python only

# Converting text into Bow
# Sample documents
docs = ['I love NLP', 'NLP is amazing', 'I love coding']

# Building vocabulary
words = set()
for doc in docs:
    tokens = doc.split()
    words.update(tokens)
# converting set to list to keep order
words = list(words) 
print("Vocabulary:", words)

# Create Bag of Words vectors
bow_vectors = []

for doc in docs:
    tokens = doc.split()
    vector = []
    for word in words:
        vector.append(tokens.count(word))  
    bow_vectors.append(vector)

# Printing vectors
for i, vec in enumerate(bow_vectors):
    print(f"Document {i+1} BoW:", vec)

2. Using the scikit-learn library

# Converting text into Bow
# from sklearn importing countvectorizer
from sklearn.feature_extraction.text import CountVectorizer
# from sklearn importing TfidVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
# sample text corpus
corpus={
    "I love NLP and Machine Learning.",
    "Machine Learning is amazing",
    "I love learning new things."
}
# creating CountVectorizer object
# CountVectorizer converts text to numbers
vectorizer=CountVectorizer()
# Fit and transform corpus, fit creates the vocabulary and transform converts document to number array
bow_matrix=vectorizer.fit_transform(corpus)
# Get feature names (unique words in vocabulary)
print("Vocabulary:",vectorizer.get_feature_names_out())
# Printing sparse matrix
print("Bow Representation:\n",bow_matrix)
# converting sparse matrix to 2D array
print("Bow Representation:\n",bow_matrix.toarray())

Implementation of TF-IDF:

1. Using Python only

# Converting text into TF-IDF

documents = [
    "I love NLP and Machine Learning",
    "Machine Learning is amazing",
    "I love learning new things"
]
docs_tokens = [doc.lower().split() for doc in documents]
print(docs_tokens)

vocab = sorted(set(word for doc in docs_tokens for word in doc))
print(vocab)

# calculating tf 
tf_docs = []
for doc in docs_tokens:              
    tf_doc = {}                      
    doc_len = len(doc)        

    for word in vocab:  
        tf_doc[word] = doc.count(word) / doc_len 
    tf_docs.append(tf_doc) 
print(tf_docs)

# calculating idf
N = len(docs_tokens)  
idf = {}

for word in vocab:
    count = sum(1 for doc in docs_tokens if word in doc)
    if count > 0:  
        idf[word] = math.log(N / count)
    else:
        idf[word] = 0

print(idf)

# calculating TF-IDF by multiplying tf*idf
tfidf_docs = []
for tf_doc in tf_docs:
    tfidf_doc = {}
    for word in vocab:
        tfidf_doc[word] = tf_doc[word] * idf[word]
    tfidf_docs.append(tfidf_doc)

for i, doc in enumerate(tfidf_docs):
    print(f"Document {i+1}: {doc}")

2. Using the scikit-learn library

# Converting text into TF-IDF
# from sklearn importing TfidVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
# using same sample text corpus
# creating object
tfidf_vectorizer=TfidfVectorizer()
# Fit the vectorizer on the corpus and transform documents into TF-IDF numeric vectors
tfidf_matrix=tfidf_vectorizer.fit_transform(corpus)
# Get feature names (unique words in vocabulary)
print("TF-IDF Vocabulary:",tfidf_vectorizer.get_feature_names_out())
# Printing sparse matrix
print("TF-IDF Representation:\n",tfidf_matrix)
# converting sparse matrix to 2D array
print("TF-IDF Representation:\n",tfidf_matrix.toarray())