D7net
Home
Console
Upload
information
Create File
Create Folder
About
Tools
:
/
usr
/
local
/
ssl
/
local
/
lib
/
python3.6
/
site-packages
/
nltk
/
app
/
Filename :
collocations_app.py
back
Copy
# Natural Language Toolkit: Collocations Application # Much of the GUI code is imported from concordance.py; We intend to merge these tools together # Copyright (C) 2001-2020 NLTK Project # Author: Sumukh Ghodke <sghodke@csse.unimelb.edu.au> # URL: <http://nltk.org/> # For license information, see LICENSE.TXT # import threading import queue as q from tkinter.font import Font from tkinter import ( Button, END, Frame, IntVar, LEFT, Label, Menu, OptionMenu, SUNKEN, Scrollbar, StringVar, Text, Tk, ) from nltk.corpus import ( cess_cat, brown, nps_chat, treebank, sinica_treebank, alpino, indian, floresta, mac_morpho, machado, cess_esp, ) from nltk.util import in_idle from nltk.probability import FreqDist CORPUS_LOADED_EVENT = "<<CL_EVENT>>" ERROR_LOADING_CORPUS_EVENT = "<<ELC_EVENT>>" POLL_INTERVAL = 100 _DEFAULT = "English: Brown Corpus (Humor)" _CORPORA = { "Catalan: CESS-CAT Corpus": lambda: cess_cat.words(), "English: Brown Corpus": lambda: brown.words(), "English: Brown Corpus (Press)": lambda: brown.words( categories=["news", "editorial", "reviews"] ), "English: Brown Corpus (Religion)": lambda: brown.words(categories="religion"), "English: Brown Corpus (Learned)": lambda: brown.words(categories="learned"), "English: Brown Corpus (Science Fiction)": lambda: brown.words( categories="science_fiction" ), "English: Brown Corpus (Romance)": lambda: brown.words(categories="romance"), "English: Brown Corpus (Humor)": lambda: brown.words(categories="humor"), "English: NPS Chat Corpus": lambda: nps_chat.words(), "English: Wall Street Journal Corpus": lambda: treebank.words(), "Chinese: Sinica Corpus": lambda: sinica_treebank.words(), "Dutch: Alpino Corpus": lambda: alpino.words(), "Hindi: Indian Languages Corpus": lambda: indian.words(files="hindi.pos"), "Portuguese: Floresta Corpus (Portugal)": lambda: floresta.words(), "Portuguese: MAC-MORPHO Corpus (Brazil)": lambda: mac_morpho.words(), "Portuguese: Machado Corpus (Brazil)": lambda: machado.words(), "Spanish: CESS-ESP Corpus": lambda: cess_esp.words(), } class CollocationsView: _BACKGROUND_COLOUR = "#FFF" # white def __init__(self): self.queue = q.Queue() self.model = CollocationsModel(self.queue) self.top = Tk() self._init_top(self.top) self._init_menubar() self._init_widgets(self.top) self.load_corpus(self.model.DEFAULT_CORPUS) self.after = self.top.after(POLL_INTERVAL, self._poll) def _init_top(self, top): top.geometry("550x650+50+50") top.title("NLTK Collocations List") top.bind("<Control-q>", self.destroy) top.protocol("WM_DELETE_WINDOW", self.destroy) top.minsize(550, 650) def _init_widgets(self, parent): self.main_frame = Frame( parent, dict(background=self._BACKGROUND_COLOUR, padx=1, pady=1, border=1) ) self._init_corpus_select(self.main_frame) self._init_results_box(self.main_frame) self._init_paging(self.main_frame) self._init_status(self.main_frame) self.main_frame.pack(fill="both", expand=True) def _init_corpus_select(self, parent): innerframe = Frame(parent, background=self._BACKGROUND_COLOUR) self.var = StringVar(innerframe) self.var.set(self.model.DEFAULT_CORPUS) Label( innerframe, justify=LEFT, text=" Corpus: ", background=self._BACKGROUND_COLOUR, padx=2, pady=1, border=0, ).pack(side="left") other_corpora = list(self.model.CORPORA.keys()).remove( self.model.DEFAULT_CORPUS ) om = OptionMenu( innerframe, self.var, self.model.DEFAULT_CORPUS, command=self.corpus_selected, *self.model.non_default_corpora() ) om["borderwidth"] = 0 om["highlightthickness"] = 1 om.pack(side="left") innerframe.pack(side="top", fill="x", anchor="n") def _init_status(self, parent): self.status = Label( parent, justify=LEFT, relief=SUNKEN, background=self._BACKGROUND_COLOUR, border=0, padx=1, pady=0, ) self.status.pack(side="top", anchor="sw") def _init_menubar(self): self._result_size = IntVar(self.top) menubar = Menu(self.top) filemenu = Menu(menubar, tearoff=0, borderwidth=0) filemenu.add_command( label="Exit", underline=1, command=self.destroy, accelerator="Ctrl-q" ) menubar.add_cascade(label="File", underline=0, menu=filemenu) editmenu = Menu(menubar, tearoff=0) rescntmenu = Menu(editmenu, tearoff=0) rescntmenu.add_radiobutton( label="20", variable=self._result_size, underline=0, value=20, command=self.set_result_size, ) rescntmenu.add_radiobutton( label="50", variable=self._result_size, underline=0, value=50, command=self.set_result_size, ) rescntmenu.add_radiobutton( label="100", variable=self._result_size, underline=0, value=100, command=self.set_result_size, ) rescntmenu.invoke(1) editmenu.add_cascade(label="Result Count", underline=0, menu=rescntmenu) menubar.add_cascade(label="Edit", underline=0, menu=editmenu) self.top.config(menu=menubar) def set_result_size(self, **kwargs): self.model.result_count = self._result_size.get() def _init_results_box(self, parent): innerframe = Frame(parent) i1 = Frame(innerframe) i2 = Frame(innerframe) vscrollbar = Scrollbar(i1, borderwidth=1) hscrollbar = Scrollbar(i2, borderwidth=1, orient="horiz") self.results_box = Text( i1, font=Font(family="courier", size="16"), state="disabled", borderwidth=1, yscrollcommand=vscrollbar.set, xscrollcommand=hscrollbar.set, wrap="none", width="40", height="20", exportselection=1, ) self.results_box.pack(side="left", fill="both", expand=True) vscrollbar.pack(side="left", fill="y", anchor="e") vscrollbar.config(command=self.results_box.yview) hscrollbar.pack(side="left", fill="x", expand=True, anchor="w") hscrollbar.config(command=self.results_box.xview) # there is no other way of avoiding the overlap of scrollbars while using pack layout manager!!! Label(i2, text=" ", background=self._BACKGROUND_COLOUR).pack( side="left", anchor="e" ) i1.pack(side="top", fill="both", expand=True, anchor="n") i2.pack(side="bottom", fill="x", anchor="s") innerframe.pack(side="top", fill="both", expand=True) def _init_paging(self, parent): innerframe = Frame(parent, background=self._BACKGROUND_COLOUR) self.prev = prev = Button( innerframe, text="Previous", command=self.previous, width="10", borderwidth=1, highlightthickness=1, state="disabled", ) prev.pack(side="left", anchor="center") self.next = next = Button( innerframe, text="Next", command=self.__next__, width="10", borderwidth=1, highlightthickness=1, state="disabled", ) next.pack(side="right", anchor="center") innerframe.pack(side="top", fill="y") self.reset_current_page() def reset_current_page(self): self.current_page = -1 def _poll(self): try: event = self.queue.get(block=False) except q.Empty: pass else: if event == CORPUS_LOADED_EVENT: self.handle_corpus_loaded(event) elif event == ERROR_LOADING_CORPUS_EVENT: self.handle_error_loading_corpus(event) self.after = self.top.after(POLL_INTERVAL, self._poll) def handle_error_loading_corpus(self, event): self.status["text"] = "Error in loading " + self.var.get() self.unfreeze_editable() self.clear_results_box() self.freeze_editable() self.reset_current_page() def handle_corpus_loaded(self, event): self.status["text"] = self.var.get() + " is loaded" self.unfreeze_editable() self.clear_results_box() self.reset_current_page() # self.next() collocations = self.model.next(self.current_page + 1) self.write_results(collocations) self.current_page += 1 def corpus_selected(self, *args): new_selection = self.var.get() self.load_corpus(new_selection) def previous(self): self.freeze_editable() collocations = self.model.prev(self.current_page - 1) self.current_page = self.current_page - 1 self.clear_results_box() self.write_results(collocations) self.unfreeze_editable() def __next__(self): self.freeze_editable() collocations = self.model.next(self.current_page + 1) self.clear_results_box() self.write_results(collocations) self.current_page += 1 self.unfreeze_editable() def load_corpus(self, selection): if self.model.selected_corpus != selection: self.status["text"] = "Loading " + selection + "..." self.freeze_editable() self.model.load_corpus(selection) def freeze_editable(self): self.prev["state"] = "disabled" self.next["state"] = "disabled" def clear_results_box(self): self.results_box["state"] = "normal" self.results_box.delete("1.0", END) self.results_box["state"] = "disabled" def fire_event(self, event): # Firing an event so that rendering of widgets happen in the mainloop thread self.top.event_generate(event, when="tail") def destroy(self, *e): if self.top is None: return self.top.after_cancel(self.after) self.top.destroy() self.top = None def mainloop(self, *args, **kwargs): if in_idle(): return self.top.mainloop(*args, **kwargs) def unfreeze_editable(self): self.set_paging_button_states() def set_paging_button_states(self): if self.current_page == -1 or self.current_page == 0: self.prev["state"] = "disabled" else: self.prev["state"] = "normal" if self.model.is_last_page(self.current_page): self.next["state"] = "disabled" else: self.next["state"] = "normal" def write_results(self, results): self.results_box["state"] = "normal" row = 1 for each in results: self.results_box.insert(str(row) + ".0", each[0] + " " + each[1] + "\n") row += 1 self.results_box["state"] = "disabled" class CollocationsModel: def __init__(self, queue): self.result_count = None self.selected_corpus = None self.collocations = None self.CORPORA = _CORPORA self.DEFAULT_CORPUS = _DEFAULT self.queue = queue self.reset_results() def reset_results(self): self.result_pages = [] self.results_returned = 0 def load_corpus(self, name): self.selected_corpus = name self.collocations = None runner_thread = self.LoadCorpus(name, self) runner_thread.start() self.reset_results() def non_default_corpora(self): copy = [] copy.extend(list(self.CORPORA.keys())) copy.remove(self.DEFAULT_CORPUS) copy.sort() return copy def is_last_page(self, number): if number < len(self.result_pages): return False return self.results_returned + ( number - len(self.result_pages) ) * self.result_count >= len(self.collocations) def next(self, page): if (len(self.result_pages) - 1) < page: for i in range(page - (len(self.result_pages) - 1)): self.result_pages.append( self.collocations[ self.results_returned : self.results_returned + self.result_count ] ) self.results_returned += self.result_count return self.result_pages[page] def prev(self, page): if page == -1: return [] return self.result_pages[page] class LoadCorpus(threading.Thread): def __init__(self, name, model): threading.Thread.__init__(self) self.model, self.name = model, name def run(self): try: words = self.model.CORPORA[self.name]() from operator import itemgetter text = [w for w in words if len(w) > 2] fd = FreqDist(tuple(text[i : i + 2]) for i in range(len(text) - 1)) vocab = FreqDist(text) scored = [ ((w1, w2), fd[(w1, w2)] ** 3 / (vocab[w1] * vocab[w2])) for w1, w2 in fd ] scored.sort(key=itemgetter(1), reverse=True) self.model.collocations = list(map(itemgetter(0), scored)) self.model.queue.put(CORPUS_LOADED_EVENT) except Exception as e: print(e) self.model.queue.put(ERROR_LOADING_CORPUS_EVENT) # def collocations(): # colloc_strings = [w1 + ' ' + w2 for w1, w2 in self._collocations[:num]] def app(): c = CollocationsView() c.mainloop() if __name__ == "__main__": app() __all__ = ["app"]