Tokenisation

import pandas as pd

content = ['hey boss!','great..','nice@']

df=pd.DataFrame(content,columns={'Sms content'})
df

import re
def tokenize(text):
    tokens=re.split('\W+',text)
    return tokens

df['tokenized_text']=df['Sms content'].apply(lambda row : tokenize(row.lower()))
df.head()