Debian Codegraph @

python/obitools3/parsers/embl.pyx

raw
1 #cython: language_level=3
2
3 '''
4 Created on June 12th 2018
5
6 @author: coissac/mercier
7 '''
8
9
10 import types
11 import re
12 import sys
13 import os
14 import glob
15
16 from obitools3.files.universalopener cimport uopen
17 from obitools3.utils cimport tostr
18 from obitools3.dms.obiseq cimport Nuc_Seq
19 from .embl_genbank_features import extractTaxon # TODO cimport
20
21 from libc.stdlib cimport free, malloc, realloc
22 from libc.string cimport strcpy, strlen
23
24
25 _featureMatcher = re.compile(b'(^FT .*\n)+', re.M)
26 _cleanFT = re.compile(b'^FT',re.M)
27
28 _headerMatcher = re.compile(b'^ID.+(?=\nFH )', re.DOTALL)
29 _seqMatcher = re.compile(b'(^ ).+(?=//\n)', re.DOTALL + re.M)
30 _cleanSeq = re.compile(b'[ \n0-9]+')
31 _acMatcher = re.compile(b'(?<=^AC ).+',re.M)
32 _deMatcher = re.compile(b'(^DE .+\n)+',re.M)
33 _cleanDe = re.compile(b'(^|\n)DE +')
34
35
36 def emblParser(bytes text):
37
38 cdef Nuc_Seq seq
39
40 try:
41 header = _headerMatcher.search(text).group()
42
43 ft = _featureMatcher.search(text).group()
44 ft = _cleanFT.sub(b' ',ft)
45
46 s = _seqMatcher.search(text).group()
47 s = _cleanSeq.sub(b'', s).upper()
48
49 acs = _acMatcher.search(text).group()
50 acs = acs.replace(b';', b' ')
51 acs = acs.split()
52 ac = acs[0]
53 acs = acs[1:]
54
55 de = _deMatcher.search(header).group()
56 de = _cleanDe.sub(b' ', de).strip().strip(b'.')
57
58 except Exception as e:
59 print("\nCould not import sequence id:", text.split()[1], "(error raised:", e, ")")
60 # Do not raise any Exception if you need the possibility to resume the generator
61 # (Python generators can't resume after any exception is raised)
62 return None
63
64 tags = {}
65 extractTaxon(ft, tags)
66
67 seq = Nuc_Seq(ac,
68 s,
69 definition=de,
70 quality=None,
71 offset=-1,
72 tags=tags)
73
74 return seq
75
76
77 def emblIterator_file(lineiterator,
78 int skip=0,
79 only=None,
80 firstline=None,
81 int buffersize=100000000
82 ):
83 cdef int lines_to_skip, ionly, read
84 cdef Nuc_Seq seq
85 cdef char* entry = NULL
86 cdef size_t entry_buffer_size
87 cdef int entry_len
88 cdef int line_len
89
90 entry_buffer_size = 2048
91
92 entry = <char*> malloc(entry_buffer_size*sizeof(char))
93
94 if only is None:
95 ionly = -1
96 else:
97 ionly = int(only)
98
99 if isinstance(lineiterator, (str, bytes)):
100 lineiterator=uopen(lineiterator)
101 if isinstance(lineiterator, LineBuffer):
102 iterator = iter(lineiterator)
103 else:
104 if hasattr(lineiterator, "readlines"):
105 iterator = iter(LineBuffer(lineiterator, buffersize))
106 elif hasattr(lineiterator, '__next__'):
107 iterator = lineiterator
108 else:
109 raise Exception("Invalid line iterator")
110
111 skipped = 0
112 read = 0
113
114 if firstline is None:
115 line = next(iterator)
116 else:
117 line = firstline
118
119 while True:
120
121 if ionly >= 0 and read >= ionly-1:
122 break
123
124 while skipped < skip:
125 line = next(iterator)
126 try:
127 while line[:2] != b'//':
128 line = next(iterator)
129 line = next(iterator)
130 except StopIteration:
131 break
132 skipped += 1
133
134 try:
135 entry_len = 0
136 while line[:2] != b'//':
137 line_len = strlen(line)
138 while (entry_len + line_len) >= entry_buffer_size:
139 entry_buffer_size*=2
140 entry = <char*>realloc(entry, entry_buffer_size)
141 strcpy(entry+entry_len, line)
142 entry_len+=line_len
143 line = next(iterator)
144 # Add last line too because need the // flag to parse
145 line_len = strlen(line)
146 while (entry_len + line_len) >= entry_buffer_size:
147 entry_buffer_size*=2
148 entry = <char*>realloc(entry, entry_buffer_size)
149 strcpy(entry+entry_len, line)
150 line = next(iterator)
151 except StopIteration:
152 break
153
154 seq = emblParser(entry)
155
156 yield seq
157 read+=1
158
159 # Last sequence
160 seq = emblParser(entry)
161
162 yield seq
163
164 free(entry)
165
166
167 def emblIterator_dir(dir_path,
168 int skip=0,
169 only=None,
170 firstline=None,
171 int buffersize=100000000
172 ):
173 path = dir_path
174 read = 0
175 read_files = 0
176 files = [filename for filename in glob.glob(os.path.join(path, b'*.dat*'))]
177 for filename in files:
178 if read==only:
179 return
180 print("Parsing file %s (%d/%d)" % (tostr(filename), read_files+1, len(files)))
181 f = uopen(filename)
182 if only is not None:
183 only_f = only-read
184 else:
185 only_f = None
186 for seq in emblIterator_file(f, skip=skip, only=only_f, buffersize=buffersize):
187 yield seq
188 read+=1
189 read_files+=1
190
191
192 def emblIterator(obj,
193 int skip=0,
194 only=None,
195 firstline=None,
196 int buffersize=100000000
197 ):
198 if type(obj) == bytes or type(obj) == str :
199 return emblIterator_dir(obj, skip=skip, only=only, firstline=firstline, buffersize=buffersize)
200 else:
201 return emblIterator_file(obj, skip=skip, only=only, firstline=firstline, buffersize=buffersize)
202
203