#NLTK library for natural language processing
import nltk
from nltk.tokenize import word_tokenize
from nltk.tokenize import wordpunct_tokenize
from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords

Sentiment Analysis

from nltk.sentiment.vader import SentimentIntensityAnalyzer
from senticnet.senticnet import SenticNet

#Dataset management/data handling
import pandas as pd
import numpy as np #for calculation
import string
import re

#Visualization
import matplotlib.pyplot as plt
import plotly.express as px
import plotly.graph_objects as go
import seaborn as sns

Suppress warnings

import warnings
warnings.filterwarnings("ignore")

Initialize SenticNet and VADER Sentiment Analyzer

sn = SenticNet()
vader = SentimentIntensityAnalyzer()

import os #for user convenience

Define the base directory

######### Change based on your path here ##########
base_dir = r'C:\Users\user\projects\Python-VS\Sentiment_Analysis\Song_csv'

List of artist names

artists = [
'ArianaGrande', 'Beyonce', 'BillieEilish', 'CardiB', 'CharliePuth',
'ColdPlay', 'Drake', 'DuaLipa', 'EdSheeran', 'Eminem', 'JustinBieber',
'KatyPerry', 'Khalid', 'LadyGaga', 'Maroon5', 'NickiMinaj',
'PostMalone', 'Rihanna', 'SelenaGomez', 'TaylorSwift'
]

Build the list of file paths

file_paths = [os.path.join(base_dir, f"{artist}.csv") for artist in artists]

Print file paths (optional)

for file in file_paths:
print(f"File path: {file}")

Read CSV files into DataFrame list

df_all = [pd.read_csv(file) for file in file_paths]

####### Data preparation ###########

Preprocessing function

Remove any song that are not the original version of an artist's songs

def cleaning(df):
a=[]
i=0
df1=df
title = df['Title']
for t in df['Title']:
r=Re=l=Li=c=m=V=ve=D=rs=edit=Edit=0
r=t.find('remix')
Re=t.find('Remix')
l=t.find('live')
Li=t.find('Live')
V=t.find('Version')
ve=t.find('version')
D=t.find('Demo ')
D=t.find('Demo')
rs=t.find('Reprise')
c=t.find('COPY')
m=t.find('Mix')
edit=t.find('edit')
Edit=t.find('Edit')
if r != -1:
a.append(t)
elif Re != -1:
a.append(t)
elif l != -1:
a.append(t)
elif Li != -1:
a.append(t)
elif V != -1:
a.append(t)
elif ve != -1:
a.append(t)
elif D != -1:
a.append(t)
elif rs != -1:
a.append(t)
elif c != -1:
a.append(t)
elif m != -1:
a.append(t)
elif edit != -1:
a.append(t)
elif Edit != -1:
a.append(t)

text
for t1 in df['Title']:
    for t2 in a:
        if t1 == t2:
            df1=df1.drop(i)
    i=i+1

df1.dropna(subset = ["Title"], inplace=True)
df1.dropna(subset = ["Lyric"], inplace=True)
df1.drop_duplicates(subset ="Title",keep = False, inplace = True)
df1.drop_duplicates(subset ="Lyric",keep = False, inplace = True) 

return df1

Change lyrics into word_list that is easier to be process during analysis

such as, making all lyrics in lowercase to make lyrics not case-sensitive

remove stop word to de-clutter the dataframe

using lemmatazation to only use base word

def lyrics_to_words(document):
if not isinstance(document, str):
return "" # Handle non-string input

text
stop_words = set(stopwords.words('english'))
exclude = set(string.punctuation)
lemma = WordNetLemmatizer()

stopwordremoval = " ".join([i for i in document.lower().split() if i not in stop_words])
punctuationremoval = ''.join(ch for ch in stopwordremoval if ch not in exclude)
normalized = " ".join(lemma.lemmatize(word) for word in punctuationremoval.split())
return normalized

Extract unique words from lyric in a song

def uniqueWords(df):
def unique(list1):
return list(set(list1)) # Simplified using set

text
words = []
for word in df['Lyric']:
    processed_lyric = lyrics_to_words(word)
    words.append(unique(processed_lyric.split()))
df['words'] = words
return df

Main pipeline

dataframe that is clean

df_allclean = []
for df in df_all:
df_cleaned = cleaning(df)
df_word_unique = uniqueWords(df_cleaned)
df_allclean.append(df_word_unique)

df_main is the dataframe that is cleaned (df_allclean)

frames = df_allclean
df_main = pd.concat(frames,ignore_index=True)
df_main.drop_duplicates(subset=['Title', 'Lyric'], inplace=True)

Remove any song that exist outside the span of 1900 and 2024

df_main['Year'] = pd.to_numeric(df_main['Year'], errors='coerce') # Ensure all values are numeric
df_main['Year'] = df_main['Year'].fillna(0) # Replace NaN with 0
df_main['Year'] = df_main['Year'].astype(int) # Convert to integer
df_main = df_main[(df_main['Year'] >= 1900) & (df_main['Year'] <= 2024)]

remove the index column (unnamed) from csv as it is not needed

for col in ['Unnamed: 0', 'Unnamed: 0.1']:
if col in df_main.columns:
df_main = df_main.drop(col, axis=1)
df_main=df_main.reset_index(drop=True)

Define a function to handle contractions

Here, word like i'm is not considered as I am but iam, this is to reduce the amout of word for each lyrics

and not to stray far from the original form of the song

def expand_contractions(text):
contractions = {
"i'm": "im", "i've": "ive", "i'll": "ill", "i'd": "id", "you're": "youre", "you've": "youve", "you'll": "youll",
"he's": "hes", "she's": "shes", "it's": "its", "they're": "theyre", "they've": "theyve", "we're": "were",
"we've": "weve", "we'll": "well", "they'll": "theyll", "isn't": "isnt", "aren't": "arent", "wasn't": "wasnt",
"weren't": "werent", "hasn't": "hasnt", "haven't": "havent", "hadn't": "hadnt", "won't": "wont", "wouldn't": "wouldnt",
"shouldn't": "shouldnt", "can't": "cant", "couldn't": "couldnt", "don't": "dont", "doesn't": "doesnt", "didn't": "didnt",
"cardi b": "cardib"
}

text
# Replace contractions with expanded forms
pattern = re.compile(r'\b(' + '|'.join(contractions.keys()) + r')\b')
expanded_text = pattern.sub(lambda x: contractions[x.group()], text)

# Remove apostrophes from other instances
expanded_text = re.sub(r"'", '', expanded_text)  # Remove all apostrophes
return expanded_text

#Function to cound word in a lyric for a song for analysis
def countword(df):
all_words = []
for lyrics in df['Lyric']:
expanded_lyrics = expand_contractions(lyrics.lower()) # Expand contractions first
words = wordpunct_tokenize(expanded_lyrics) # Tokenize
all_words.extend(words)
return all_words

'''
########### Basic Analysis / Comparison cleaned dataframe with original #############

Lists to store information

before = []
after = []
length = []
ulength = []
wd = []

Populate before and after counts, and compute word statistics

for df in df_all:
a, b = df.shape # Get the number of rows (a) and columns (b) before cleaning
before.append(a)

for dfc in df_allclean:
a, b = dfc.shape # Get the number of rows (a) and columns (b) after cleaning
after.append(a)
c = countword(dfc)
l = len(c)
ul = len(np.unique(c)) # Unique words
wd.append(c)
length.append(l)
ulength.append(ul)

artists = ['Ariana Grande', 'Beyoncé', 'Billie Eilish', 'Cardi B', 'Charlie Puth', 'Coldplay', 'Drake', 'Dua Lipa',
'Ed Sheeran', 'Eminem', 'Justin Bieber', 'Katy Perry', 'Khalid', 'Lady Gaga', 'Maroon 5', 'Nicki Minaj',
'Post Malone', 'Rihanna', 'Selena Gomez', 'Taylor Swift']

df_info =pd.DataFrame({'name':artists,'before':before,'after':after,'words':wd,'unique words':ulength,'word count':length})
df_info['diff']=df_info['before']-df_info['after']
df_info['words per songs'] = round(df_info['word count'] / df_info['after'],0)
df_info['words per songs'] = df_info['words per songs'].astype('int')
df_info['lexicalrichness']=(df_info['unique words']/df_info['word count'])*100
df_info=df_info[['name','before','after','diff','words','words per songs','unique words','word count','lexicalrichness']]

The analysis for words between cleaned data and original data.

print(df_info)

#Visualitzaion of these analysis

Unique Word VS Total Word Graph for Each Artist (Bar Graph)

fig = go.Figure(data=[
go.Bar(name='Unique Word Count', x=df_info['name'], y=df_info['unique words'].tolist()),
go.Bar(name='Total Word Count', x=df_info['name'], y=df_info['word count'].tolist()),
])
fig.update_layout(barmode='group',title={'text': "Total words vs Unique words",'y':0.9,'x':0.5,'xanchor': 'center','yanchor': 'top'})
fig.show()

Lexicalrichness of all artist based on their songs lyrics

fig = px.bar(df_info, x='name',y='lexicalrichness')
fig.update_layout(title={'text': "Lexicalrichness of all artist",'y':1,'x':0.5,'xanchor': 'center','yanchor': 'top'})
fig.show()

Grouping data by Artist and Year to see trend

df_group = df_main.groupby(['Artist', 'Year']).agg({'Title': 'count', 'Lyric': 'count'}).reset_index()
df_group.rename(columns={'Title': 'Title_Count', 'Lyric': 'Song_Count'}, inplace=True)

Further Grouping: Total Songs per Year Across All Artists

df_temp = df_group.groupby(['Year']).agg({'Title_Count': 'sum'}).reset_index()

Visualization: Number of Songs by All Artists from 1900-2024

fig = px.line(df_temp, x='Year', y='Title_Count',
title="Number of Titles by All Artists from 2001-2024",
labels={'Year': 'Year', 'Title_Count': 'Number of Titles'},
markers=True)

fig.update_layout(
title={
'text': "Number of Titles by All Artists from 1900-2024",
'y':0.95,
'x':0.5,
'xanchor': 'center',
'yanchor': 'top'
},
xaxis_title="Year",
yaxis_title="Number of Titles"
)
fig.show()

########## Sentiment Analysis ###########

VADER Sentiment Analyzer function

def setimentanalyzer(df):
# Scores that is negative is considered to give result of negative emotion such as sadness, anger, sarcasm
# Scores that is neutral nearing 0 is considered to give result of neutral emotion such as sadness, anger
# Scores that is positive is considered to give result of positive emotion such as happiness, proud
neg='Negative'
neu='Neutral'
pos='Positive'
negative = []
neutral = []
positive = []
dominant_sentiment=[]
dominant_sentiment_score=[]
#Initialize the model
sid = SentimentIntensityAnalyzer()
#Iterate for each row of lyrics and append the scores
for i in df.index:

text
    scores = sid.polarity_scores(df['Lyric'].iloc[i])
    negative.append(scores['neg'])
    neutral.append(scores['neu'])
    positive.append(scores['pos'])
    if scores['neg']>scores['pos']:
        dominant_sentiment_score.append(scores['neg'])
        dominant_sentiment.append(neg)
    elif scores['neg']<scores['pos']:
        dominant_sentiment_score.append(scores['pos'])
        dominant_sentiment.append(pos)
    else:
        dominant_sentiment_score.append(scores['neu'])
        dominant_sentiment.append(neu)
#Create 5 columns to the main data frame  for each score
df['negative'] = negative
df['neutral'] = neutral
df['positive'] = positive
df['dominant_sentiment']=dominant_sentiment
df['dominant_sentiment_score']=dominant_sentiment_score
return df

df_sentiment = setimentanalyzer(df_main)
print(df_main)
#df_sentiment.to_csv("VADER_Sentiment_Analyzer.csv", index=False)

#Visualization for Sentimentanalysis using VADER Sentiment Analyzer

Temporary storage for artist data

df_temp = []
artists = df_sentiment['Artist'].unique()

Ensure we have enough unique artists for the plots

if len(artists) < 20:
raise ValueError("Not enough unique artists for the plots.")

Subset data for each artist and store in df_temp

for artist in artists:
df_temp.append(df_sentiment[df_sentiment['Artist'] == artist])

Define a custom color palette

custom_palette = {
'Positive': 'green',
'Negative': 'red',
'Neutral': 'blue'
}

Ensure consistent order for the categories

category_order = ['Positive', 'Negative', 'Neutral']

First 9 artists (3x3 grid)

fig, axs = plt.subplots(nrows=3, ncols=3, figsize=(15, 15), constrained_layout=True)

for i in range(9):
row = i // 3
col = i % 3
sns.swarmplot(data=df_temp[i], x="dominant_sentiment", y="dominant_sentiment_score",
ax=axs[row][col], palette=custom_palette, order=category_order)
axs[row][col].set_title(artists[i])
axs[row][col].set_xlabel("Dominant Sentiment")
axs[row][col].set_ylabel("Sentiment Score")

Next 9 artists (3x3 grid)

fig, axs = plt.subplots(nrows=3, ncols=3, figsize=(15, 15), constrained_layout=True)

for i in range(9, 18):
row = (i - 9) // 3
col = (i - 9) % 3
sns.swarmplot(data=df_temp[i], x="dominant_sentiment", y="dominant_sentiment_score",
ax=axs[row][col], palette=custom_palette, order=category_order)
axs[row][col].set_title(artists[i])
axs[row][col].set_xlabel("Dominant Sentiment")
axs[row][col].set_ylabel("Sentiment Score")

Last 2 artists (3x1 grid)

fig, axs = plt.subplots(nrows=1, ncols=3, figsize=(15, 5), constrained_layout=True)

for i in range(18, 20):
col = i - 18
sns.swarmplot(data=df_temp[i], x="dominant_sentiment", y="dominant_sentiment_score",
ax=axs[col], palette=custom_palette, order=category_order)
axs[col].set_title(artists[i])
axs[col].set_xlabel("Dominant Sentiment")
axs[col].set_ylabel("Sentiment Score")

Remove unused axes in the last grid

for col in range(2, 3): # Indices 2 and beyond are unused
axs[col].set_visible(False)

plt.show()

'''

function to analyze song lyrics sentiment value using SenticNet

def analyze_sentiment(word_list):
# Initialize SenticNet
sn = SenticNet()

text
# Dictionary to store polarity and primary emotion for each word
sentiment_data = {}

# Variable to count words not found in SenticNet
count_of_word_not_in_senticNet = 0

# Loop over each word in the list
for word in word_list:
    try:
        # Attempt to get SenticNet data for the word
        sn_data = sn.concept(word)

        mood_tags = sn.moodtags(word)
        first_mood_tag = mood_tags[0]
        second_mood_tag = mood_tags[1]
        
        if sn_data:
            
            # Extract polarity and primary mood from SenticNet data
            polarity = sn_data.get('polarity_value', 'N/A')

            # Store the sentiment data in the dictionary
            sentiment_data[word] = {
                'polarity': polarity,
                'primary_mood': first_mood_tag,
                'secondary_mood': second_mood_tag
            }
    except Exception as e:
        # Increment the count if an error occurs
        count_of_word_not_in_senticNet += 1

return sentiment_data, count_of_word_not_in_senticNet

fixed_moods based on all emotion in sectinet6

fixed_moods = {
"#eagerness": 0,
"#calmness": 0,
"#joy": 0,
"#pleasantess": 0,
"#disgust": 0,
"#sadness": 0,
"#anger": 0,
"#fear": 0
}

def songSentiment_analyzer_by_word_primary_emotion(artist):
# filter the song by Artist
Artist_df = df_main[df_main.Artist == artist]

text
# Get the total number of songs
total_songs = Artist_df.shape[0]
#print(f"Total number of songs: {total_rows}")

song_mood_category = fixed_moods.copy()

number_of_word_not_in_artist_lyric = 0
for song in range(total_songs):
    row_ = Artist_df.iloc[song]
    text = row_.words
    sentiment_analysis, count = analyze_sentiment(text)

    mood_count = fixed_moods.copy()

    for word, data in sentiment_analysis.items():
        primary_emotion = data['primary_mood']

        if primary_emotion in mood_count:
            mood_count[primary_emotion] += 1
        pass
        #print(f"Word: {word} - Polarity: {data['polarity']} - Primary Emotion: {data['primary_mood']}")

    #find the most frequent mood that all word in a song represent
    highest_mood = max(mood_count, key=mood_count.get)

    if highest_mood in song_mood_category:
        song_mood_category[highest_mood] += 1
    number_of_word_not_in_artist_lyric += count

#print(song_mood_category)
#print(f"Words not found in SenticNet for all song by {artist}: {number_of_word_not_in_artist_lyric}")
#print("")
return total_songs, song_mood_category, number_of_word_not_in_artist_lyric

polarity_list = []

def songSentiment_analyzer_by_polarity_averaging(artist):
# filter the song by Artist
Artist_df = df_main[df_main.Artist == artist]

text
# Get the total number of songs
total_songs = Artist_df.shape[0]
#print(f"Total number of songs: {total_rows}")

polarity_list_for_avgSONG = polarity_list.copy()
avg_polarity_list_for_artist = polarity_list.copy()

number_of_word_not_in_artist_lyric = 0
for song in range(total_songs): #total_songs
    row_ = Artist_df.iloc[song]
    text = row_.words
    sentiment_analysis, count = analyze_sentiment(text)

    for word, data in sentiment_analysis.items():
        polarity = data['polarity']
        polarity = float(polarity) #senticnet polarity value is in string

        #print(polarity)
        polarity_list_for_avgSONG.append(polarity)

    average_polarity_song = np.mean(polarity_list_for_avgSONG)
    #print(average_polarity_song)
    avg_polarity_list_for_artist.append(average_polarity_song)

avg_polarity_for_artist = np.mean(avg_polarity_list_for_artist)

return avg_polarity_for_artist

pass

artists = ['Ariana Grande', 'Beyoncé', 'Billie Eilish', 'Cardi B', 'Charlie Puth', 'Coldplay', 'Drake', 'Dua Lipa',
'Ed Sheeran', 'Eminem', 'Justin Bieber', 'Katy Perry', 'Khalid', 'Lady Gaga', 'Maroon 5', 'Nicki Minaj',
'Post Malone', 'Rihanna', 'Selena Gomez', 'Taylor Swift']

Initialize a list to hold results

df_secticnet = []

for artist in artists:
# Perform sentiment analysis for the current artist
total_songs, song_mood_category, number_of_word_not_in_artist_lyric = songSentiment_analyzer_by_word_primary_emotion(artist)

text
# Create a dictionary with the artist's data
artist_data = {
    "Artist": artist,
    "Total Songs": total_songs,
    "Words Not Found": number_of_word_not_in_artist_lyric
}

# Add mood categories to the dictionary
artist_data.update(song_mood_category)

# Append the artist's data to the list
df_secticnet.append(artist_data)

Create a DataFrame from the list of dictionaries

df_senticnet = pd.DataFrame(df_secticnet)

print(df_senticnet)

Visualization Mode-baseed approach of SenticNet

'''
mood_colors = {
'#eagerness': '#e66f6f', # Slightly darker red
'#calmness': '#3380cc', # Slightly darker blue
'#joy': '#66cc66', # Slightly darker green
'#pleasantess': '#e6b36e', # Slightly darker orange
'#disgust': '#8f8fdb', # Slightly darker lavender
'#sadness': '#e680b3', # Slightly darker pink
'#anger': '#80cc80', # Slightly darker light green
'#fear': '#e64d4d', # Slightly darker red
'Other': '#b3b3b3' # Slightly darker gray
}

Iterate over each artist and plot a pie chart for their mood distribution

for index, row in df_senticnet.iterrows():
# Extract the mood distribution for the artist
mood_data = row[['#eagerness', '#calmness', '#joy', '#pleasantess', '#disgust', '#sadness', '#anger', '#fear']]

text
# Calculate the total sum of the mood data
total = mood_data.sum()

# Calculate percentages
percentages = (mood_data / total) * 100

# Group values below 2% into an "Other" category
filtered_data = mood_data[percentages >= 2]
other_value = mood_data[percentages < 2].sum()

# Add the "Other" category if it has a value
if other_value > 0:
    filtered_data['Other'] = other_value

# Sort filtered_data to maintain order in pie chart
filtered_data = filtered_data.sort_index()

# Prepare colors for the filtered data
colors = [mood_colors[mood] for mood in filtered_data.index]

# Plot the pie chart
plt.figure(figsize=(6, 6))
plt.pie(
    filtered_data,
    labels=filtered_data.index,
    autopct=lambda p: f'{p:.1f}%' if p > 2 else '',
    startangle=140,
    colors=colors
)
plt.title(f"Mood Distribution for {row['Artist']}")
plt.show()

'''

Initialize a list to hold results

df_secticnet_polarity = []

for artist in artists:
# Perform sentiment analysis for the current artist
avg_polarity_for_artist = songSentiment_analyzer_by_polarity_averaging(artist)

text
# Create a dictionary with the artist's data
artist_data = {
    "Artist": artist,
    "Average Polarity Score": avg_polarity_for_artist
}

# Append the artist's data to the list
df_secticnet_polarity.append(artist_data)

Create a DataFrame from the list of dictionaries

df_secticnet_polarity = pd.DataFrame(df_secticnet_polarity)

Display the DataFrame

print(df_secticnet_polarity)

Visualization of SenticNet Polarity Score

plt.figure(figsize=(12, 6))
plt.bar(df_secticnet_polarity["Artist"], df_secticnet_polarity["Average Polarity Score"], color='skyblue')
plt.xticks(rotation=45, ha='right') # Rotate artist names for better readability
plt.xlabel("Artists")
plt.ylabel("Average Polarity Score")
plt.title("Average Polarity Score by Artist")
plt.tight_layout()

Show the plot

plt.show()

can you identify the flow of this program? starting from data collection etc, letsay there is no data collection then no need tomention