HMM实现中文分词
【摘要】 import numpy as np
import warnings
from hmmlearn.hmm import MultinomialHMM as mhmm
data=[{
u"我要吃饭":"SSBE"},
{
u"天气不错" : "BEBE"},
{
u"谢天谢地" : "BMME"}]
def prints(s): pass print(s)
def ge...
import numpy as np
import warnings
from hmmlearn.hmm import MultinomialHMM as mhmm
data=[{
u"我要吃饭":"SSBE"},
{
u"天气不错" : "BEBE"},
{
u"谢天谢地" : "BMME"}]
def prints(s): pass print(s)
def get_startprob(): """get BMES matrix """ c=0 c_map={"B":0,"M":0,"E":0,"S":0} #caculate the count for v in data : for key in v : value=v[key] c=c+1 prints("value[0] is "+value[0]) c_map[value[0]]=c_map[value[0]] +1 prints("c_map[value[0]] is "+str(c_map[value[0]]) ) res=[] for i in "BMES": res.append( c_map[i] / float(c)) return res
def get_transmat(): """get transmat of status """ c=0 #record BE:1,BB:2 c_map={} for v in data : for key in v : value=v[key] prints("value[0] is "+value[0]) for v_i in range(len(value)-1): couple=value[v_i:v_i+2] c_couple_source = c_map.get(couple,0) c_map[couple]=c_couple_source+1 c=c+1 #c_map[value[0]]=c_map[value[0]] +1 #prints("c_map[value[0]] is "+str(c_map[value[0]]) ) prints("get_transmat's c_map is "+str(c_map)) res=[] for i in "BMES": col=[] col_count=0 for j in "BMES": col_count=c_map.get(i+j,0)+col_count for j in "BMES": col.append( c_map.get(i+j,0) / float(col_count)) res.append(col) return res
def get_words(): return u"我要吃饭天气不错谢天地"
def get_word_map(): words=get_words() res={} for i in range(len(words)): res[words[i]]=i return res
def get_array_from_phase(phase): word_map=get_word_map() res=[] for key in phase: res.append(word_map[key]) return res
def get_emissionprob(): #get emmissionprob of status and observers c=0 #record Bc=0 #record B我:1,B吃:2 c_map={} for v in data : for key in v : k=key value=v[key] prints("value[0] is "+value[0]) for v_i in range(len(value)): couple=value[v_i]+k[v_i] prints("emmition's couple is " + couple) c_couple_source = c_map.get(couple,0) c_map[couple]=c_couple_source+1 c=c+1 res=[] prints("emmition's c_map is "+str(c_map)) words=get_words() for i in "BMES": col=[] for j in words: col.append( c_map.get(i+j,0) / float(c)) res.append(col) return res
if( __name__ == "__main__"): # print("startprob is ",get_startprob()) # print("transmat is " ,get_transmat()) print("emissionprob is " , get_emissionprob()) print("word map is ",get_word_map()) # coding=utf-8 warnings.filterwarnings("ignore") # import matplotlib.pyplot as plt startprob = np.array(get_startprob()) print("startprob is ", startprob) transmat = np.array(get_transmat()) print("transmat is ", transmat) emissionprob = np.array(get_emissionprob()) print("emmissionprob is ", emissionprob) mul_hmm = mhmm(n_components=4) mul_hmm.startprob_ = startprob mul_hmm.transmat_ = transmat mul_hmm.emissionprob_ = emissionprob phase = u"我要吃饭谢天谢地" X = np.array(get_array_from_phase(phase)) X = X.reshape(len(phase), 1) print("X is ", X) Y = mul_hmm.predict(X) print("Y is ", Y) # {B（词开头），M（词中），E（词尾），S（独字词）} {0,1,2,3}
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
- 48
- 49
- 50
- 51
- 52
- 53
- 54
- 55
- 56
- 57
- 58
- 59
- 60
- 61
- 62
- 63
- 64
- 65
- 66
- 67
- 68
- 69
- 70
- 71
- 72
- 73
- 74
- 75
- 76
- 77
- 78
- 79
- 80
- 81
- 82
- 83
- 84
- 85
- 86
- 87
- 88
- 89
- 90
- 91
- 92
- 93
- 94
- 95
- 96
- 97
- 98
- 99
- 100
- 101
- 102
- 103
- 104
- 105
- 106
- 107
- 108
- 109
- 110
- 111
- 112
- 113
- 114
- 115
- 116
- 117
- 118
- 119
- 120
- 121
- 122
- 123
- 124
- 125
out
F:\anaconda\pythonw.exe D:/学习资料/网易云课堂/唐宇迪-机器学习课程(新)/自然语言处理(Python版)/第八章：HMM实战/HMM案例实战/HMM/get_hmm_param.py
value[0] is S
emmition's couple is S我
emmition's couple is S要
emmition's couple is B吃
emmition's couple is E饭
value[0] is B
emmition's couple is B天
emmition's couple is E气
emmition's couple is B不
emmition's couple is E错
value[0] is B
emmition's couple is B谢
emmition's couple is M天
emmition's couple is M谢
emmition's couple is E地
emmition's c_map is {'S我': 1, 'S要': 1, 'B吃': 1, 'E饭': 1, 'B天': 1, 'E气': 1, 'B不': 1, 'E错': 1, 'B谢': 1, 'M天': 1, 'M谢': 1, 'E地': 1}
emissionprob is [[0.0, 0.0, 0.08333333333333333, 0.0, 0.08333333333333333, 0.0, 0.08333333333333333, 0.0, 0.08333333333333333, 0.08333333333333333, 0.0], [0.0, 0.0, 0.0, 0.0, 0.08333333333333333, 0.0, 0.0, 0.0, 0.08333333333333333, 0.08333333333333333, 0.0], [0.0, 0.0, 0.0, 0.08333333333333333, 0.0, 0.08333333333333333, 0.0, 0.08333333333333333, 0.0, 0.0, 0.08333333333333333], [0.08333333333333333, 0.08333333333333333, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]
word map is {'我': 0, '要': 1, '吃': 2, '饭': 3, '天': 9, '气': 5, '不': 6, '错': 7, '谢': 8, '地': 10}
value[0] is S
c_map[value[0]] is 1
value[0] is B
c_map[value[0]] is 1
value[0] is B
c_map[value[0]] is 2
startprob is [0.66666667 0. 0. 0.33333333]
value[0] is S
value[0] is B
value[0] is B
get_transmat's c_map is {'SS': 1, 'SB': 1, 'BE': 3, 'EB': 1, 'BM': 1, 'MM': 1, 'ME': 1}
transmat is [[0. 0.25 0.75 0. ]
[0. 0.5 0.5 0. ]
[1. 0. 0. 0. ]
[0.5 0. 0. 0.5 ]]
value[0] is S
emmition's couple is S我
emmition's couple is S要
emmition's couple is B吃
emmition's couple is E饭
value[0] is B
emmition's couple is B天
emmition's couple is E气
emmition's couple is B不
emmition's couple is E错
value[0] is B
emmition's couple is B谢
emmition's couple is M天
emmition's couple is M谢
emmition's couple is E地
emmition's c_map is {'S我': 1, 'S要': 1, 'B吃': 1, 'E饭': 1, 'B天': 1, 'E气': 1, 'B不': 1, 'E错': 1, 'B谢': 1, 'M天': 1, 'M谢': 1, 'E地': 1}
emmissionprob is [[0. 0. 0.08333333 0. 0.08333333 0.
0.08333333 0. 0.08333333 0.08333333 0. ]
[0. 0. 0. 0. 0.08333333 0.
0. 0. 0.08333333 0.08333333 0. ]
[0. 0. 0. 0.08333333 0. 0.08333333
0. 0.08333333 0. 0. 0.08333333]
[0.08333333 0.08333333 0. 0. 0. 0.
0. 0. 0. 0. 0. ]]
X is [[ 0]
[ 1]
[ 2]
[ 3]
[ 8]
[ 9]
[ 8]
[10]]
Y is [3 3 0 2 0 1 1 2]
Process finished with exit code 0
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
- 48
- 49
- 50
- 51
- 52
- 53
- 54
- 55
- 56
- 57
- 58
- 59
- 60
- 61
- 62
- 63
- 64
- 65
- 66
- 67
- 68
- 69
- 70
- 71
文章来源: maoli.blog.csdn.net，作者：刘润森！，版权归原作者所有，如需转载，请联系作者。
原文链接：maoli.blog.csdn.net/article/details/89440323
【版权声明】本文为华为云社区用户转载文章，如果您发现本社区中有涉嫌抄袭的内容，欢迎发送邮件进行举报，并提供相关证据，一经查实，本社区将立刻删除涉嫌侵权内容，举报邮箱：
cloudbbs@huaweicloud.com
- 点赞
- 收藏
- 关注作者
评论（0）