Debian Codegraph @

texmf-dist/source/support/optexcount/word_iterator.py

raw
1 import re
2
3
4 class WordIterator:
5 """
6 A basic wrapper for sequential reading of words from file
7 """
8
9 def __init__(self, filename):
10 self.filename = filename
11 self.word_queue = []
12 self.line_payload = 50 # how many lines will be loaded into the queue at one time
13 self.__lines_count = sum(1 for line in open(filename))
14 self.__load_line = 1 # index of first line which haven't been already loaded into the queue
15 self.__separators = []
16
17 def read(self):
18 """
19 Returns first word from file that wasn't already read by this particular object.
20 Return value format: (word, separator)
21 When all words had been read, it returns None
22 """
23 if len(self.word_queue) == 0:
24 self.__load_payload()
25 if len(self.word_queue) == 0:
26 return None
27 else:
28 return self.word_queue.pop(0)
29
30 def push_back(self, word):
31 """
32 Pushes word from argument into the first position of the queue
33 In case we already read a word but we want to re-read it again
34 """
35 self.word_queue = [word] + self.word_queue
36
37 def add_separator(self, sep):
38 """
39 All words will be separated with sep separator
40 """
41 if sep in self.__separators:
42 return
43 self.__separators.append(sep)
44 new_queue = []
45 regex_arg = re.compile('(' + sep + ')')
46 for word, sep in self.word_queue:
47 new_word = re.sub(regex_arg, r' \1 ', word)
48 new_words = new_word.split(' ')
49 new_queue += self.__make_pairs(word + sep, new_words)
50 self.word_queue = new_queue
51
52 def __load_payload(self):
53 """
54 Loads words from input file into the queue
55 Format of words: (word, separator)
56 """
57 index = 0
58 with open(self.filename, 'r') as file:
59 for line in file:
60 index += 1
61 if index < self.__load_line:
62 continue
63 elif index < self.__load_line + self.line_payload:
64 self.word_queue += self.__parse_words(line)
65 if index >= self.__lines_count:
66 index += 1
67 else:
68 break
69 self.__load_line = index
70
71 def __parse_words(self, line):
72 """
73 Parses a line passed by argument using regular expressions.
74 It separates each word on the line and stores it into list.
75 returns list of tuples (word, separator between word and next word)
76 """
77 orig_line = line
78 if line == "\n" or line == "\r\n":
79 return self.__make_pairs(orig_line, [line])
80 # All non escape occurrences of some characters are seperated to be a single 'word'
81 line = re.sub(r'(?<!\\)(?:\\\\)*([{}\[\]()%])', r' \1 ', line)
82 # $$ and $ are separated from the text to be a single 'word'
83 line = re.sub(r'(?<!\\)(?:\\\\)*((\$\$)|(\$))', r' \1 ', line)
84 # All escaped alphabetic characters are separated from the previous word
85 line = re.sub(r'(?<!\\)(\\\\)*(\\)([A-Za-z])', r'\1 \2\3', line)
86 # Non escaped character ~ is replaced with space
87 line = re.sub(r'(?<!\\)(?:\\\\)*(~)', r' \1 ', line)
88 # Iterate through every separator and put spaces around them
89 for sep in self.__separators:
90 line = re.sub('(' + sep + ')', r' \1 ', line)
91 words_on_line = re.split(r'\s+', line)
92 # At the end of every nonempty line newline character is placed
93 words_on_line.append("\n")
94 return self.__make_pairs(orig_line, words_on_line)
95
96 @staticmethod
97 def __make_pairs(orig_line, words):
98 """
99 Generates list of tuples (word, separator between the word and the next word in list)
100 """
101 pairs = []
102 read_index = 0
103 for i in range(len(words)):
104 if i + 1 >= len(words):
105 start = orig_line[read_index:].find(words[i]) + len(words[i]) + read_index
106 pairs.append((words[i], orig_line[start:]))
107 else:
108 start = orig_line[read_index:].find(words[i]) + len(words[i]) + read_index
109 end = orig_line[start:].find(words[i+1]) + start
110 pairs.append((words[i], orig_line[start: end]))
111 read_index = end
112 return pairs