HMM实现中文分词
【摘要】 import numpy as np
import warnings
from hmmlearn.hmm import MultinomialHMM as mhmm
data=[{
u"我要吃饭":"SSBE"},
{
u"天气不错" : "BEBE"},
{
u"谢天谢地" : "BMME"}]
def prints(s): pass print(s)
def ge...
import numpy as np
import warnings
from hmmlearn.hmm import MultinomialHMM as mhmm
data=[{
u"我要吃饭":"SSBE"},
{
u"天气不错" : "BEBE"},
{
u"谢天谢地" : "BMME"}]
def prints(s): pass print(s)
def get_startprob(): """get BMES matrix """ c=0 c_map={"B":0,"M":0,"E":0,"S":0} #caculate the count for v in data : for key in v : value=v[key] c=c+1 prints("value[0] is "+value[0]) c_map[value[0]]=c_map[value[0]] +1 prints("c_map[value[0]] is "+str(c_map[value[0]]) ) res=[] for i in "BMES": res.append( c_map[i] / float(c)) return res
def get_transmat(): """get transmat of status """ c=0 #record BE:1,BB:2 c_map={} for v in data : for key in v : value=v[key] prints("value[0] is "+value[0]) for v_i in range(len(value)-1): couple=value[v_i:v_i+2] c_couple_source = c_map.get(couple,0) c_map[couple]=c_couple_source+1 c=c+1 #c_map[value[0]]=c_map[value[0]] +1 #prints("c_map[value[0]] is "+str(c_map[value[0]]) ) prints("get_transmat's c_map is "+str(c_map)) res=[] for i in "BMES": col=[] col_count=0 for j in "BMES": col_count=c_map.get(i+j,0)+col_count for j in "BMES": col.append( c_map.get(i+j,0) / float(col_count)) res.append(col) return res
def get_words(): return u"我要吃饭天气不错谢天地"
def get_word_map(): words=get_words() res={} for i in range(len(words)): res[words[i]]=i return res
def get_array_from_phase(phase): word_map=get_word_map() res=[] for key in phase: res.append(word_map[key]) return res
def get_emissionprob(): #get emmissionprob of status and observers c=0 #record Bc=0 #record B我:1,B吃:2 c_map={} for v in data : for key in v : k=key value=v[key] prints("value[0] is "+value[0]) for v_i in range(len(value)): couple=value[v_i]+k[v_i] prints("emmition's couple is " + couple) c_couple_source = c_map.get(couple,0) c_map[couple]=c_couple_source+1 c=c+1 res=[] prints("emmition's c_map is "+str(c_map)) words=get_words() for i in "BMES": col=[] for j in words: col.append( c_map.get(i+j,0) / float(c)) res.append(col) return res
if( __name__ == "__main__"): # print("startprob is ",get_startprob()) # print("transmat is " ,get_transmat()) print("emissionprob is " , get_emissionprob()) print("word map is ",get_word_map()) # coding=utf-8 warnings.filterwarnings("ignore") # import matplotlib.pyplot as plt startprob = np.array(get_startprob()) print("startprob is ", startprob) transmat = np.array(get_transmat()) print("transmat is ", transmat) emissionprob = np.array(get_emissionprob()) print("emmissionprob is ", emissionprob) mul_hmm = mhmm(n_components=4) mul_hmm.startprob_ = startprob mul_hmm.transmat_ = transmat mul_hmm.emissionprob_ = emissionprob phase = u"我要吃饭谢天谢地" X = np.array(get_array_from_phase(phase)) X = X.reshape(len(phase), 1) print("X is ", X) Y = mul_hmm.predict(X) print("Y is ", Y) # {B（词开头），M（词中），E（词尾），S（独字词）} {0,1,2,3}
out
F:\anaconda\pythonw.exe D:/学习资料/网易云课堂/唐宇迪-机器学习课程(新)/自然语言处理(Python版)/第八章：HMM实战/HMM案例实战/HMM/get_hmm_param.py
value[0] is S
emmition's couple is S我
emmition's couple is S要
emmition's couple is B吃
emmition's couple is E饭
value[0] is B
emmition's couple is B天
emmition's couple is E气
emmition's couple is B不
emmition's couple is E错
value[0] is B
emmition's couple is B谢
emmition's couple is M天
emmition's couple is M谢
emmition's couple is E地
emmition's c_map is {'S我': 1, 'S要': 1, 'B吃': 1, 'E饭': 1, 'B天': 1, 'E气': 1, 'B不': 1, 'E错': 1, 'B谢': 1, 'M天': 1, 'M谢': 1, 'E地': 1}
emissionprob is [[0.0, 0.0, 0.08333333333333333, 0.0, 0.08333333333333333, 0.0, 0.08333333333333333, 0.0, 0.08333333333333333, 0.08333333333333333, 0.0], [0.0, 0.0, 0.0, 0.0, 0.08333333333333333, 0.0, 0.0, 0.0, 0.08333333333333333, 0.08333333333333333, 0.0], [0.0, 0.0, 0.0, 0.08333333333333333, 0.0, 0.08333333333333333, 0.0, 0.08333333333333333, 0.0, 0.0, 0.08333333333333333], [0.08333333333333333, 0.08333333333333333, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]
word map is {'我': 0, '要': 1, '吃': 2, '饭': 3, '天': 9, '气': 5, '不': 6, '错': 7, '谢': 8, '地': 10}
value[0] is S
c_map[value[0]] is 1
value[0] is B
c_map[value[0]] is 1
value[0] is B
c_map[value[0]] is 2
startprob is [0.66666667 0. 0. 0.33333333]
value[0] is S
value[0] is B
value[0] is B
get_transmat's c_map is {'SS': 1, 'SB': 1, 'BE': 3, 'EB': 1, 'BM': 1, 'MM': 1, 'ME': 1}
transmat is [[0. 0.25 0.75 0. ]
[0. 0.5 0.5 0. ]
[1. 0. 0. 0. ]
[0.5 0. 0. 0.5 ]]
value[0] is S
emmition's couple is S我
emmition's couple is S要
emmition's couple is B吃
emmition's couple is E饭
value[0] is B
emmition's couple is B天
emmition's couple is E气
emmition's couple is B不
emmition's couple is E错
value[0] is B
emmition's couple is B谢
emmition's couple is M天
emmition's couple is M谢
emmition's couple is E地
emmition's c_map is {'S我': 1, 'S要': 1, 'B吃': 1, 'E饭': 1, 'B天': 1, 'E气': 1, 'B不': 1, 'E错': 1, 'B谢': 1, 'M天': 1, 'M谢': 1, 'E地': 1}
emmissionprob is [[0. 0. 0.08333333 0. 0.08333333 0.
0.08333333 0. 0.08333333 0.08333333 0. ]
[0. 0. 0. 0. 0.08333333 0.
0. 0. 0.08333333 0.08333333 0. ]
[0. 0. 0. 0.08333333 0. 0.08333333
0. 0.08333333 0. 0. 0.08333333]
[0.08333333 0.08333333 0. 0. 0. 0.
0. 0. 0. 0. 0. ]]
X is [[ 0]
[ 1]
[ 2]
[ 3]
[ 8]
[ 9]
[ 8]
[10]]
Y is [3 3 0 2 0 1 1 2]
Process finished with exit code 0
文章来源: maoli.blog.csdn.net，作者：刘润森！，版权归原作者所有，如需转载，请联系作者。
原文链接：maoli.blog.csdn.net/article/details/89440323
【版权声明】本文为华为云社区用户转载文章，如果您发现本社区中有涉嫌抄袭的内容，欢迎发送邮件进行举报，并提供相关证据，一经查实，本社区将立刻删除涉嫌侵权内容，举报邮箱：
cloudbbs@huaweicloud.com
- 点赞
- 收藏
- 关注作者
作者其他文章
评论（0）