推荐 | 文本分析库cntext2.x使用说明文档

cd desktop

pip3 install pdfdocx
pip3 install distinctiveness
pip3 install cntext-2.1.1-py3-none-any.whl

import cntext as ct

help(ct)

Help on package cntext:

NAME
    cntext

PACKAGE CONTENTS
    io
    mind
    model
    stats
...

import cntext as ct

ct.get_dict_list()

['zh_common_NTUSD.yaml',
 'zh_common_DUTIR.yaml',
 'enzh_common_StopWords.yaml',
 'en_valence_Concreteness.yaml',
 'en_common_LoughranMcDonald.yaml',
 'zh_common_FinanceSenti.yaml',
 'zh_common_FLS.yaml',
 'zh_common_TsinghuaPraiseDegrade.yaml',
 'zh_common_FEPU.yaml',
 'en_common_ANEW.yaml',
 'en_common_NRC.yaml',
 'zh_valence_ChineseEmoBank.yaml',
 'zh_valence_SixSemanticDimensionDatabase.yaml',
 'zh_common_FinacialFormalUnformal.yaml',
 'zh_common_LoughranMcDonald.yaml',
 'enzh_common_AdvConj.yaml',
 'en_common_SentiWS.yaml',
 'zh_common_Digitalization.yaml',
 'en_common_LSD2015.yaml',
 'zh_common_HowNet.yaml',
 'zh_common_EPU.yaml']

import cntext as ct
print(ct.read_yaml_dict('zh_common_Digitalization.yaml'))

{'Name': '中文数字化词典', 
'Desc': '基于这篇论文，构建了中文数字化词典，含人工智能技术、大数据技术、云计算技术、区块链技术、数字技术应用等关键词列表。 ', 'Refer': '吴非,胡慧芷,林慧妍,任晓怡. 企业数字化转型与资本市场表现——来自股票流动性的经验证据[J]. 管理世界,2021,37(07):130-144+10.', 
'Category': ['Artificial_Intelligence', 'Big_Data', 'Cloud_Computing', 'Block_Chains', 'Usage_of_Digitalization'], 

'Dictionary': 
    {'Artificial_Intelligence': ['人工智能', '商业智能', '图像理解', '投资决策辅助系统', '智能数据分析', '智能机器人', '机器学习', '深度学习', '语义搜索', '生物识别技术', '人脸识别', '语音识别', '身份验证', '自动驾驶', '自然语言处理'], 
    'Big_Data': ['大数据', '数据挖掘', '文本挖掘', '数据可视化', '异构数据', '征信', '增强现实', '混合现实', '虚拟现实'], 
    'Cloud_Computing': ['云计算', '流计算', '图计算', '内存计算', '多方安全计算', '类脑计算', '绿色计算', '认知计算', '融合架构', '亿级并发', 'EB级存储', '物联网', '信息物理系统'], 
    'Block_Chains': ['区块链', '数字货币', '分布式计算', '差分隐私技术', '智能金融合约'], 
    'Usage_of_Digitalization': ['移动互联网', '工业互联网', '移动互联', '互联网医疗', '电子商务', '移动支付', '第三方支付', 'NFC支付', '智能能源', 'B2B', 'B2C', 'C2B', 'C2C', 'O2O', '网联', '智能穿戴', '智慧农业', '智能交通', '智能医疗', '智能客服', '智能家居', '智能投顾', '智能文旅', '智能环保', '智能电网', '智能营销', '数字营销', '无人零售', '互联网金融', '数字金融', 'Fintech', '金融科技', '量化金融', '开放银行']}}

ct.detect_encoding(file, num_lines=100)

import cntext as ct

#读取data文件夹下的【三体.txt】
#识别编码方式
ct.detect_encoding(file='data/三体.txt', num_lines=100)

utf-8

ct.read_file(file, encoding='utf-8')

import cntext as ct

#默认encoding='utf-8'
#sdf = ct.read_file(fformat='data/三体.txt')

sdf = ct.read_file(fformat='data/三体.txt', encoding='utf-8')
sdf

#查看【文件夹data】内所有的 txt文件。
ct.get_files(fformat='data/*.txt')

['data/三体.txt',
 'data/santi.txt',
 'data/w2v_corpus.txt',
 'data/sopmi_corpus.txt',
 'data/brown_corpus.txt',
 'data/sopmi_seed_words.txt']

ct.read_files(fformat, encoding='utf-8'）

import cntext as ct

#默认encoding='utf-8'
#ddf = ct.read_files(fformat='data/*.txt')

ddf = ct.read_files(fformat='data/*.txt', encoding='utf-8')
ddf

term_freq(text, lang='chinese', return_df=False)

import cntext as ct

text = '致力于致力于以零文章处理费或订阅费发布优质研究软件。'

#ct.term_freq(text, lang='chinese')
ct.term_freq(text)

Counter({'致力于': 2,
         '文章': 1,
         '处理费': 1,
         '订阅费': 1,
         '发布': 1,
         '优质': 1,
         '研究': 1,
         '软件': 1})

ct.term_freq(text, return_df=True)

ct.readability(text, lang='chinese')

import cntext as ct

text1 = '致力于以零文章处理费或订阅费发布优质研究软件。'

ct.readability(text1, lang='chinese')

{'readability1': 23.0, 'readability2': 2.0, 'readability3': 12.5}

sentiment(text, diction, lang='chinese', return_df=False)

import cntext as ct

diction = {'pos': ['高兴', '快乐', '分享'],
           'neg': ['难过', '悲伤'],
           'adv': ['很', '特别']}

text = '我今天得奖了，很高兴，我要将快乐分享大家。'
ct.sentiment(text=text, 
             diction=diction, 
             lang='chinese')

{'pos_num': 3,
 'neg_num': 0,
 'adv_num': 1,
 'stopword_num': 8,
 'word_num': 14,
 'sentence_num': 1}

import cntext as ct

diction = {'pos': ['高兴', '快乐', '分享'],
           'neg': ['难过', '悲伤'],
           'adv': ['很', '特别']}

text = '我今天得奖了，很高兴，我要将快乐分享大家。'

ct.sentiment(text=text, 
             diction=diction, 
             lang='chinese', 
             return_df=True)

ct.sentiment_by_valence(text, diction, lang='chinese', return_df=False)

import cntext as ct
import pandas as pd

concreteness_dict = ct.read_yaml_dict('en_valence_Concreteness.yaml')['Dictionary']
concreteness_dict

{'roadsweeper': {'concreteness': 4.85},
 'traindriver': {'concreteness': 4.54},
 'tush': {'concreteness': 4.45},
 'hairdress': {'concreteness': 3.93},
 'pharmaceutics': {'concreteness': 3.77},
 'hoover': {'concreteness': 3.76},
 'shopkeeping': {'concreteness': 3.18},
 'pushiness': {'concreteness': 2.48},
 ......
 }

reply = "I'll go look for that"

score=ct.sentiment_by_valence(text=reply, 
                              diction=concreteness_dict, 
                              lang='english')

score

{'text': "I'll go look for that", 
'concreteness': 9.28, 
'word_num': 6}

employee_replys = ["I'll go look for that",
                   "I'll go search for that",
                   "I'll go search for that top",
                   "I'll go search for that t-shirt",
                   "I'll go look for that t-shirt in grey",
                   "I'll go search for that t-shirt in grey"]

for idx, reply in enumerate(employee_replys):
    score=ct.sentiment_by_valence(text=reply, 
                                  diction=concreteness_dict, 
                                  lang='english')
    
    template = "Concreteness Score: {score:.2f} | Example-{idx}: {exmaple}"
    
    print(template.format(score=score['concreteness'], 
                          idx=idx, 
                          exmaple=reply))

Concreteness Score: 9.28 | Example-0: I'll go look for that
Concreteness Score: 9.32 | Example-1: I'll go search for that
Concreteness Score: 13.25 | Example-2: I'll go search for that top
Concreteness Score: 14.25 | Example-3: I'll go search for that t-shirt
Concreteness Score: 21.32 | Example-4: I'll go look for that t-shirt in grey
Concreteness Score: 21.36 | Example-5: I'll go search for that t-shirt in grey

ct.word_in_context(text, keywords, window=3, lang='chinese')

import cntext as ct

#测试代码，假设zh_text是年报文本，从找找出丝网词相关词的上下文
zh_text = """
【插入一条自家广告】大邓自己家的家，
安平县多隆丝网制品，生产销售不锈钢轧花网、
电焊网、石笼网、刀片刺绳、冲孔网等丝网制品。
联系人 邓颖静 0318-7686899

人生苦短，我学Python
在社科中，可以用Python做文本分析
Python是一门功能强大的编程语言，广泛应用在经管社科领域。
可以做网络爬虫、文本分析、LDA话题模型、相似度分析等。

今年经济不景气，形势异常严峻。
由于疫情不景气，静默管理， 产品积压， 公司经营困难。
保就业促就业，任务十分艰巨。
"""

#【python】上下文
ct.word_in_context(text = zh_text, 
                   keywords = ['python'], 
                   window=10, 
                   lang='chinese')

ct.epu(df, e_pattern, p_pattern, u_pattern)

import cntext as ct

#省略，读取数据得到 news_df

epu_df = ct.epu(df=news_df, freq='M')
epu_df

ct.fepu(text, ep_pattern, u_pattern)

import cntext as ct

#省略，读取数据得到 mda_df

fepu_df = df['经营讨论与分析内容'].apply(ct.fepu)
res_df = pd.concat([df[['会计年度', '股票代码']], fepu_df],   axis=1)
res_df

ct.semantic_brand_score(text, brands, lang='chinese')

import cntext as ct

brands = ['汪淼', '史强', '罗辑', '叶文洁', '伊文斯']

#准备santi_test_text
#小说等分20份， 读取第一份得到santi_test_text

sbs_df = ct.semantic_brand_score(text=santi_test_text, 
                               brands=brands, 
                               lang='chinese')
sbs_df

ct.cosine_sim(text1, text2)   cos余弦相似
ct.jaccard_sim(text1, text2)  jaccard相似
ct.minedit_sim(text1, text2)  最小编辑距离相似度； 
ct.simple_sim(text1, text2)   更改变动算法

import cntext as ct 


text1 = '编程真好玩编程真好玩'
text2 = '游戏真好玩编程真好玩'

print('cosine: ', ct.cosine_sim(text1, text2))
print('jaccard', ct.jaccard_sim(text1, text2))
print('minedit', ct.minedit_sim(text1, text2))
print('simple', ct.simple_sim(text1, text2))

cosine:  0.82
jaccard: 0.67
minedit: 1.00
simple:  0.84

ct.W2VModel(corpus_file,  encoding='utf-8', lang='chinese') 

import cntext as ct

#训练模型
#[data/三体.txt]体积2.7M
w2v = ct.W2VModel(corpus_file='data/三体.txt',  #语料txt文件路径
                  encoding='utf-8',#语料txt文件编码方式
                  lang='chinese') #英文传english

w2v.train() #设置存储

Start Training! This may take a while. Please be patient...

Training word2vec model took 5 seconds

Note: The Word2Vec model has been saved to output/Word2Vec

ct.Glove(corpus_file, lang='chinese')

import cntext as ct
import os

model = ct.Glove(corpus_file='data/三体.txt', 
                 lang='chinese')
model.train()

Create vocabulary for Glove.

Create cooccurrence matrix.

Create cooccurrence matrix.
To complete this task, the code may take a significant amount of time, ranging from several minutes to potentially hours. Please be patient while the process runs.

Iteration 20: error 10541294.8481
Finish training! Used 22.38 s

Save the glove embeddings to a binary file

ct.SoPmi(corpus_file, seed_file)       #人工标注的初始种子词

import cntext as ct

sopmier = ct.SoPmi(corpus_file='data/sopmi_corpus.txt',   
                   seed_file='data/sopmi_seed.txt')       #人工标注的初始种子词
                     

sopmier.train()

Step 1/4:...Preprocess   Corpus ...
Step 2/4:...Collect co-occurrency information ...
Step 3/4:...Calculate   mutual information ...
Step 4/4:...Save    candidate words ...
Finish! used 19.74 s

ct.load_w2v(w2v_path)

import cntext as ct

#使用gensim也可读取训练的模型
#from gensim.models import KeyedVectors
#santi_w2v = KeyedVectors.load('output/Word2Vec/三体.100.6.bin')

santi_w2v = ct.load_w2v(w2v_path='output/Word2Vec/三体.100.6.bin')
santi_w2v

Loading word2vec model...
<gensim.models.word2vec.Word2Vec at 0x1069c0dd0>

santi_w2v.wv

<gensim.models.keyedvectors.KeyedVectors at 0x319f4a090>

ct.expand_dictionary(wv,  seeddict, topn=100)

seeddict = {
    '人物': ['叶文洁', '史强', '罗辑'], 
    '物体': ['飞船', '车辆']
}


ct.expand_dictionary(wv=santi_w2v.wv,  
                     seeddict=seeddict, 
                     topn=10)

ct.sematic_distance(wv, words, c_words1, c_words2) 

male_concept = ['male', 'man', 'he', 'him']
female_concept = ['female', 'woman', 'she', 'her']
software_engineer_concept  = ['engineer',  'programming',  'software']
d1 = distance(male_concept,  software_engineer_concept)
d2 = distance(female_concept,  software_engineer_concept)

import cntext as ct

# glove_w2v.6B.100d.txt链接: https://pan.baidu.com/s/1MMfQ7M0YCzL9Klp4zrlHBw 提取码: 72l0 
g_wv = KeyedVectors.load_word2vec_format('glove_w2v.6B.100d.txt', no_header=False)
#g_wv是gensim.models.keyedvectors.KeyedVectors

engineer = ['program', 'software', 'computer']
man_words =  ["man", "he", "him"]
woman_words = ["woman", "she", "her"]

#在语义空间中，工程师更接近于男人，而不是女人。
#in semantic space, engineer is closer to man, other than woman.
ct.sematic_distance(wv=g_wv,
                    words=engineer, 
                    c_words1=man_words, 
                    c_words2=woman_words)

-0.38

ct.sematic_projection(wv, words, c_words1, c_words2) 

animals = ['mouse', 'cat', 'horse',  'pig', 'whale']
small_words= ["small", "little", "tiny"]
large_words = ["large", "big", "huge"]

#wiki_wv = 导入wiki的模型。
#wiki_wv

# In size conception, mouse is smallest, horse is biggest.
# 在大小概念上，老鼠最小，马是最大的。
ct.sematic_projection(wv=wiki_wv,
                      words=animals, 
                      c_words1=small_words, 
                      c_words2=large_words)

[('mouse', -1.68),
 ('cat', -0.92),
 ('pig', -0.46),
 ('whale', -0.24),
 ('horse', 0.4)]

ct.divergent_association_task(wv, words)

low_words = ["arm", "eyes", "feet", "hand", "head", "leg", "body"]
average_words = ["bag", "bee", "burger", "feast", "office", "shoes", "tree"]
high_words = ["hippo", "jumper", "machinery", "prickle", "tickets", "tomato", "violin"]

# 导入模型，得到wv。
# wv为gensim.models.keyedvectors.KeyedVectors类型

print(ct.divergent_association_task(wv, low_words)) # 50
print(ct.divergent_association_task(wv, average_words)) # 78
print(ct.divergent_association_task(wv, high_words)) # 95

50
78
95

ct.procrustes_align(base_embed, other_embed)

Deng X., Nan P. (2022). cntext: a Python tool for text mining. DOI: 10.5281/zenodo.7063523 URL: https://github.com/hiDaDeng/cntext

@misc{YourReferenceHere,
author = {Deng, Xudong and Nan, Peng},
doi = {10.5281/zenodo.7063523},
month = {9},
title = {cntext: a Python tool for text mining},
url = {https://github.com/hiDaDeng/cntext},
year = {2022}
}

%0 Generic
%A Deng, Xudong
%A Nan, Peng
%D 2022
%K text mining
%K text analysi
%K social science
%K management science
%K semantic analysis
%R 10.5281/zenodo.7063523
%T cntext: a Python tool for text mining
%U https://github.com/hiDaDeng/cntext