Python自然语言处理入门

>>> from nltk.corpus import gutenberg

>>> from nltk import FreqDist

# 作图需要 matplotlib（可以从 NLTK 下载页获得）

>>> import matplotlib

>>> import matplotlib.pyplot as plt

# 统计 Gutenberg 中每个词例数量

>>> fd = FreqDist()

>>> for text in gutenberg.fileids():

... for word in gutenberg.words(text):

... fd.inc(word)

# 初始化两个空列表来存放词序和词频

>>> ranks = []

>>> freqs = []

# 生成每个词例的（词序，词频）点并且将其添加到相应列表中，

# 注意循环中的 fd 会自动排序

>>> for rank, word in enumerate(fd):

... ranks.append(rank+1)

... freqs.append(fd[word])

...

# 在 log-log 图中展示词序和词频的关系

>>> plt.loglog(ranks, freqs)

>>> plt.xlabel(’frequency(f)’, fontsize=14, fontweight=’bold’)

>>> plt.ylabel(’rank(r)’, fontsize=14, fontweight=’bold’)

>>> plt.grid(True)

>>> plt.show()

>>> from nltk.corpus import gutenberg

>>> from nltk import ConditionalFreqDist

>>> from random import choice

# 分布实例化

>>> cfd = ConditionalFreqDist()

# 对于每个实例，统计给定词的下一个词数量

>>> prev_word = None

>>> for word in gutenberg.words(’austen-persuasion.txt’):

... cfd[prev_word].inc(word)

... prev_word = word

# 给定“therefore”作为给定词作为预测器的初始词

>>> word = ’therefore’

>>> i = 1

# 找到给定词的所有下一个可能的词，并随机选择一个

>>> while i < 20:

... print word,

... lwords = cfd[word].samples()

... follower = choice(lwords)

... word = follower

... i += 1

...

therefore it known of women ought. Leave me so well

placed in five altogether well placed themselves delighted

>>> from nltk.corpus import brown

>>> from nltk import FreqDist, ConditionalFreqDist

>>> fd = FreqDist()

>>> cfd = ConditionalFreqDist()

# 对于语料库中每个标注过的句子，以（词例，词性标签）对形式表示并统计词性标签和词例的词性标签

>>> for sentence in brown.tagged_sents():

... for (token, tag) in sentence:

... fd.inc(tag)

... cfd[token].inc(tag)

>>> fd.max() # 频率最高的词性标签是 ...

’NN’

>>> wordbins = [] # Initialize a list to hold (numtags,word) tuple

# 添加每个（词例的唯一词性标签，词例）元组到列表中

>>> for token in cfd.conditions():

... wordbins.append((cfd[token].B(), token))

...

# 按唯一词性标签数从高到低对元组进行排序

>>> wordbins.sort(reverse=True)

>>> print wordbins[0] # 标签最多的词例是 ...

(12, ’that’)

>>> male = [’he’,’his’,’him’,’himself’] # 男性代词

>>> female = [’she’,’hers’,’her’,’herself’] # 女性代词

>>> n_male, n_female = 0, 0 # 初始化计数器

# 男性代词总计：

>>> for m in male:

... n_male += cfd[m].N()

...

# 女性代词总计：

>>> for f in female:

... n_female += cfd[f].N()

...

>>> print float(n_male)/n_female # 计算比率

3.257

>>> n_ambiguous = 0

>>> for (ntags, token) in wordbins:

... if ntags > 1:

... n_ambiguous += 1

...

>>> n_ambiguous # 词性标签归属多于一个的词例数

8729

>>> from nltk.corpus import brown, stopwords

>>> from nltk import ConditionalFreqDist

>>> cfd = ConditionalFreqDist()

# 得到英文停用词表

>>> stopwords_list = stopwords.words(’english’)

# 定义一个函数，如果属于名词类则返回true

>>> def is_noun(tag):

... return tag.lower() in [’nn’,’nns’,’nn$’,’nn-tl’,’nn+bez’,

’nn+hvz’, ’nns$’,’np’,’np$’,’np+bez’,’nps’,

’nps$’,’nr’,’np-tl’,’nrs’,’nr$’]

...

# 统计前 5 个单词的出现次数

>>> for sentence in brown.tagged_sents():

... for (index, tagtuple) in enumerate(sentence):

... (token, tag) = tagtuple

... token = token.lower()

... if token not in stopwords_list and is_noun(tag):

... window = sentence[index+1:index+5]

... for (window_token, window_tag) in window:

... window_token = window_token.lower()

... if window_token not in stopwords_list and 

is_noun(window_tag):

... cfd[token].inc(window_token)

# 好了。我们完成了！让我们开始进行联想！

>>> print cfd[’left’].max()

right

>>> print cfd[’life’].max()

death

>>> print cfd[’man’].max()

woman

>>> print cfd[’woman’].max()

world

>>> print cfd[’boy’].max()

girl

>>> print cfd[’girl’].max()

trouble

>>> print cfd[’male’].max()

female

>>> print cfd[’ball’].max()

player

>>> print cfd[’doctor’].max()

bills

>>> print cfd[’road’].max()

block