金融研究 | 使用Python构建「关键审计事项信息含量」

import pandas as pd

# converters 强制声明该列为字符串，  防止股票代码 被程序识别为数字，
df = pd.read_excel('mda2020.xlsx', converters={'股票代码': str})

#显示前5行
df.head()

df['行业代码'].value_counts()

ind_codes = df['行业代码'].value_counts()
ind_codes = ind_codes[ind_codes>20].index
ind_codes

def filter_industry(ind_code):
    try:
        if ind_code in ind_codes:
            return True
        else:
            return False
    except:
        return False


df2 = df[df['行业代码'].apply(filter_industry)]
df2.head()

import cntext as ct
import jieba
import re

#pip3 install cntext==1.8.6
stopwords = ct.load_pkl_dict('STOPWORDS.pkl')['STOPWORDS']['chinese']


def transform(text):
    #只保留md&a中的中文内容
    text = ''.join(re.findall('[\u4e00-\u9fa5]+', text))
    #剔除停用词
    words = [w for w in jieba.cut(text) if w not in stopwords]
    #整理为用空格间隔的字符串(类西方语言文本格式)
    return ' '.join(words)

df2['clean_text'] = df2['经营讨论与分析内容'].apply(transform)
df2.head()

from sklearn.feature_extraction.text import TfidfVectorizer

cv = TfidfVectorizer(min_df=0.05, max_df=0.5) 
# 生成稀疏bow矩阵
#dtm 文档-词频-矩阵
dtm2020 = cv.fit_transform(df2['clean_text']) 
dtm2020 = pd.DataFrame(dtm2020.toarray())
dtm2020.head()

#合并多个字段为新的df
dtm2020_ = pd.concat([df2[['股票代码', '会计年度', '行业代码']], dtm2020], axis=1)
dtm2020_.head()

import os
import pandas as pd
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np
import csv

with open('信息含量2020.csv', 'a+', newline='', encoding='utf-8') as csvf:
    fieldnames = ['股票代码', '会计年度', '行业代码', '信息含量']
    writer = csv.DictWriter(csvf, fieldnames=fieldnames)
    writer.writeheader()
    
    for idx in range(len(dtm2020_)):
        try:
            code = dtm2020_.loc[idx, '股票代码']
            ind = dtm2020_.loc[idx, '行业代码']
            year = dtm2020_.loc[idx, '会计年度']
            #企业向量
            corp_vec = [dtm2020_.iloc[idx, 3:].values]
            corp_arr = np.array(corp_vec)
            #行业向量
            ind_vec = [dtm2020_[dtm2020_['行业代码']==ind][dtm2020_['股票代码']!=code].iloc[:, 3:].mean(axis=0).values]
            ind_arr = np.array(ind_vec)
            #信息含量
            special_info = -1 * cosine_similarity(corp_arr, ind_arr)[0][0]

            data = dict()
            data['股票代码'] = code
            data['行业代码'] = ind
            data['会计年度'] = year
            data['信息含量'] = special_info

            writer.writerow(data)
        except:
            pass

import pandas as pd

idf = pd.read_csv('信息含量2020.csv')
print(len(idf))
idf.head()