|
- # -*- encoding:utf-8 -*-
- """
- * Copyright (C) 2017 OwnThink.
- *
- * Name : textrank.py - 解析
- * Author : zengbin93 <zeng_bin8888@163.com>
- * Version : 0.01
- * Description : TextRank算法实现
- special thanks to https://github.com/ArtistScript/FastTextRank
- """
- import sys
- import numpy as np
- from jiagu import utils
- from heapq import nlargest
- from collections import defaultdict
- from itertools import count, product
-
-
- class Keywords(object):
- def __init__(self,
- use_stopword=True,
- stop_words_file=utils.default_stopwords_file(),
- max_iter=100,
- tol=0.0001,
- window=2):
- self.__use_stopword = use_stopword
- self.__max_iter = max_iter
- self.__tol = tol
- self.__window = window
- self.__stop_words = set()
- self.__stop_words_file = utils.default_stopwords_file()
- if stop_words_file:
- self.__stop_words_file = stop_words_file
- if use_stopword:
- with open(self.__stop_words_file, 'r', encoding='utf-8') as f:
- for word in f:
- self.__stop_words.add(word.strip())
- np.seterr(all='warn')
-
- @staticmethod
- def build_vocab(sents):
- word_index = {}
- index_word = {}
- words_number = 0
- for word_list in sents:
- for word in word_list:
- if word not in word_index:
- word_index[word] = words_number
- index_word[words_number] = word
- words_number += 1
- return word_index, index_word, words_number
-
- @staticmethod
- def create_graph(sents, words_number, word_index, window=2):
- graph = [[0.0 for _ in range(words_number)] for _ in range(words_number)]
- for word_list in sents:
- for w1, w2 in utils.combine(word_list, window):
- if w1 in word_index and w2 in word_index:
- index1 = word_index[w1]
- index2 = word_index[w2]
- graph[index1][index2] += 1.0
- graph[index2][index1] += 1.0
- return graph
-
- def keywords(self, text, n):
- text = text.replace('\n', '')
- text = text.replace('\r', '')
- text = utils.as_text(text)
- tokens = utils.cut_sentences(text)
- sentences, sents = utils.psegcut_filter_words(tokens,
- self.__stop_words,
- self.__use_stopword)
-
- word_index, index_word, words_number = self.build_vocab(sents)
- graph = self.create_graph(sents, words_number,
- word_index, window=self.__window)
- scores = utils.weight_map_rank(graph, max_iter=self.__max_iter,
- tol=self.__tol)
- sent_selected = nlargest(n, zip(scores, count()))
- sent_index = []
- for i in range(n):
- sent_index.append(sent_selected[i][1])
- return [index_word[i] for i in sent_index]
-
-
- class Summarize(object):
- def __init__(self, use_stopword=True,
- stop_words_file=None,
- dict_path=None,
- max_iter=100,
- tol=0.0001):
- if dict_path:
- raise RuntimeError("True")
- self.__use_stopword = use_stopword
- self.__dict_path = dict_path
- self.__max_iter = max_iter
- self.__tol = tol
-
- self.__stop_words = set()
- self.__stop_words_file = utils.default_stopwords_file()
- if stop_words_file:
- self.__stop_words_file = stop_words_file
- if use_stopword:
- for word in open(self.__stop_words_file, 'r', encoding='utf-8'):
- self.__stop_words.add(word.strip())
- np.seterr(all='warn')
-
- def filter_dictword(self, sents):
- _sents = []
- dele = set()
- for sentence in sents:
- for word in sentence:
- if word not in self.__word2vec:
- dele.add(word)
- if sentence:
- _sents.append([word for word in sentence if word not in dele])
- return _sents
-
- def summarize(self, text, n):
- text = text.replace('\n', '')
- text = text.replace('\r', '')
- text = utils.as_text(text)
- tokens = utils.cut_sentences(text)
- sentences, sents = utils.cut_filter_words(tokens, self.__stop_words, self.__use_stopword)
-
- graph = self.create_graph(sents)
- scores = utils.weight_map_rank(graph, self.__max_iter, self.__tol)
- sent_selected = nlargest(n, zip(scores, count()))
- sent_index = []
- for i in range(n):
- sent_index.append(sent_selected[i][1])
- return [sentences[i] for i in sent_index]
-
- @staticmethod
- def create_graph(word_sent):
- num = len(word_sent)
- board = [[0.0 for _ in range(num)] for _ in range(num)]
-
- for i, j in product(range(num), repeat=2):
- if i != j:
- board[i][j] = utils.sentences_similarity(word_sent[i], word_sent[j])
- return board
-
- def compute_similarity_by_avg(self, sents_1, sents_2):
- if len(sents_1) == 0 or len(sents_2) == 0:
- return 0.0
- vec1 = self.__word2vec[sents_1[0]]
- for word1 in sents_1[1:]:
- vec1 = vec1 + self.__word2vec[word1]
-
- vec2 = self.__word2vec[sents_2[0]]
- for word2 in sents_2[1:]:
- vec2 = vec2 + self.__word2vec[word2]
-
- similarity = utils.cosine_similarity(vec1 / len(sents_1),
- vec2 / len(sents_2))
- return similarity
-
-
- class TextRank:
- d = 0.85
-
- def __init__(self):
- self.graph = defaultdict(list)
-
- def add_edge(self, start, end, weight=1):
- self.graph[start].append((start, end, weight))
- self.graph[end].append((end, start, weight))
-
- def rank(self):
- ws = defaultdict(float)
- out_sum = defaultdict(float)
-
- wsdef = 1.0 / (len(self.graph) or 1.0)
- for n, out in self.graph.items():
- ws[n] = wsdef
- out_sum[n] = sum((e[2] for e in out), 0.0)
-
- sorted_keys = sorted(self.graph.keys())
- for x in range(10):
- for n in sorted_keys:
- s = 0
- for e in self.graph[n]:
- s += e[2] / out_sum[e[1]] * ws[e[1]]
- ws[n] = (1 - self.d) + self.d * s
-
- min_rank, max_rank = sys.float_info[0], sys.float_info[3]
- for w in ws.values():
- if w < min_rank:
- min_rank = w
- if w > max_rank:
- max_rank = w
-
- for n, w in ws.items():
- ws[n] = (w - min_rank / 10.0) / (max_rank - min_rank / 10.0)
-
- return ws
|
