import pandas as pd
df = pd.read_csv('http://www.guofei.site/datasets/nlp/SMSSpamCollection.csv', sep='\t', header=None, names=['label', 'documents'])
y = ((df.label == 'spam') * 1).values
documents_raw = df.documents.values

import re

regex = re.compile('[a-zA-Z]{2,}')  # 2次以上字母组合，去除标点符号和数字
documents = [regex.findall(document.lower()) for document in documents_raw]
documents = [' '.join(words) for words in documents]  # 重新整合成句子
# documents = [' '.join([word for word in words if word not in stopwords]) for words in documents]  # 如果有stopswords的话


# train_test_split
from sklearn.model_selection import train_test_split
documents_train, documents_test, y_train, y_test = train_test_split(documents, y, test_size=0.2)

import pandas as pd

df = pd.read_csv('https://guofei.site/datasets/nlp/food_sentiment_cn.csv')
y = df.label
documents_raw = df.documents.values

import jieba
import re

documents = [jieba.lcut(document, cut_all=False) for document in documents_raw]  # 拆词
regex = re.compile(u'[\u4e00-\u9fa5]')  # 仅保留中文，(剔除 数字、英文)
documents = [[word for word in document if regex.match(word) is not None] for document in documents]
documents = [[word for word in document if word not in stopwords] for document in documents]  # 停词
documents = [' '.join(document) for document in documents]

import collections


class VocabularyProcessor:
    '''
    author: guofei
    site: www.guofei.site
    输入：一个列表，列表中的元素是句子，句子中的单词用空格分开。这里不提供数据清洗功能，所以应当先进行数据清洗，然后使用这个类
    输出：一个列表的列表，长度是词语个数，元素是单词的序号，例如 [[1, 2], [1, 2, 0, -1, -1, 3], [-1, 0, -1, -1], [2, 1, 0, -1, 3]]
    ```
    corpus = ['this is the first document',
              'this is the second second document',
              'and the third one',
              'is this the first document']
    vp = VocabularyProcessor(max_vocabulary_size=5)
    vp.fit(corpus)
    X = vp.transform(corpus)
    ```
    - 剔除低频词，并且把所有的低频词标记为 'RARE'，其序号固定为 0
    - max_vocabulary_size=10000 最多保留多少单词
    '''
    def __init__(self, max_vocabulary_size=10000):
        self.max_vocabulary_size = max_vocabulary_size
        self.word_count = None
        self.dictionary = None
        self.reverse_dictionary = None

    def fit(self, documents):
        words = [word for sentence in documents for word in sentence.split(' ')]
        word_count_ = collections.Counter(words)  # 词频
        word_count = word_count_.most_common(self.max_vocabulary_size - 1)  # 最多词频

        words_most = [word for word, num in word_count]  # 频率最高的词
        len_words_most=len(words_most)
        dictionary = dict(zip(words_most, range(1,len_words_most)))
        reverse_dictionary = dict(zip(range(1,len_words_most), words_most))

        # 添加 'RARE' ,也就是低频词语
        rare_count = sum([word_count_[word] for word in word_count_ if word not in words_most])
        word_count = [('RARE', rare_count)] + word_count
        dictionary['RARE'] = 0
        reverse_dictionary[0] = 'RARE'

        self.word_count = word_count
        self.dictionary = dictionary
        self.reverse_dictionary = reverse_dictionary

    def transform(self, documents):
        return [[self.dictionary.get(word, 0) for word in sentence.split(' ')] for sentence in documents]

    def fit_transform(self, documents):
        self.fit(documents)
        return self.transform(documents)

corpus = ['this is the first document',
          'this is the second second document',
          'and the third one',
          'is this the first document']
vp = VocabularyProcessor(max_vocabulary_size=5)
vp.fit(corpus)
X = vp.transform(corpus)
# vp.word_count, vp.dictionary, vp.reverse_dictionary
print(X)

vp.word_count  # 字典大小
vp.dictionary  # {'word': idx}
vp.reverse_dictionary  # {idx: 'word'}

['document', 'first', 'is', 'second', 'the', 'third', 'this']

count_vectorizer.transform(['This is the first first first document.'])
# 结果：[[1, 3, 1, 0, 1, 0, 1]]

count_vectorizer.transform(['Is this the third document?'])
# 结果： [[1, 0, 1, 0, 1, 1, 1]]

from sklearn.feature_extraction import text

count_vectorizer = text.CountVectorizer()
# ngram_range
# max_df : float in range [0.0, 1.0] or int, default=1.0，浓度超过这个值的单词会被忽略
# min_df : float in range [0.0, 1.0] or int, default=1，数量低于这个值的单词背忽略
# max_features : int or None, default=None，选取最高频的 k 个单词
# vocabulary : Mapping or iterable，允许手动指定单词
# 或者：
# text.CountVectorizer(max_df=0.95, min_df=2,
# max_features=n_features,
# stop_words='english',
# token_pattern=r'\b\S+\b'
# )

# 源数据
documents = ['This is the first document.',
             'This is the second second document.',
             'The third document.',
             'Is this the first document?']

# 训练
count_vectorizer.fit(documents)

# 预测
X = count_vectorizer.transform(documents)
# 1. 不在 vocabulary 中的词，在 transform 阶段被忽略
# 2. X 是稀疏矩阵，用 X.toarray() 转换成普通矩阵
# 或者 count_vectorizer.fit_transform(documents)

X.toarray()  # 是一个shape=(num_sentence,num_words) 的array

count_vectorizer.vocabulary_  # vocabulary向量（dict格式，{'word': idx}）
# {'this': 6, 'is': 2, 'the': 4, 'first': 1, 'document': 0, 'second': 3, 'third': 5}

count_vectorizer.get_feature_names_out()
# ['document', 'first', 'is', 'second', 'the', 'third', 'this']

ngram_range=(1, 2) # 分词时，除了自身外，还可以保留前后单词，例如，这个案例可以以这个为词典：
# ['bi', 'grams', 'are', 'cool', 'bi grams', 'grams are', 'are cool']

text.CountVectorizer(max_df=0.99, min_df=2,
                                        # max_features=n_features,
                                        stop_words='english')

from sklearn.feature_extraction import text
tf_idf_transformer=text.TfidfTransformer()

counts = [[3, 0, 1],
           [2, 0, 0],
           [3, 0, 0],
           [4, 0, 0],
           [3, 2, 0],
           [3, 0, 2]]
tf_idf_transformer.fit(counts)

tf_idf_transformer.transform(counts).toarray()

from sklearn.feature_extraction import text

tf_idf=text.TfidfVectorizer()
tf_idf.fit(documents)

# tf_idf.vocabulary_ # {'单词':idx}
# tf_idf.fixed_vocabulary_ # 是否指定 vocabulary
# tf_idf.idf_ # 每个单词的 idf 值
# tf_idf.stop_words_

X = tf_idf.transform(documents)

text.HashingVectorizer  # 占用内存更少，索引速度更快
VectorizerMixin