Debian Codegraph @

python/obitools3/parsers/genbank.pyx

raw
1 #cython: language_level=3
2
3 '''
4 Created on June 12th 2018
5
6 @author: coissac/mercier
7 '''
8
9
10 import types
11 import re
12 import sys
13 import os
14 import glob
15
16 from obitools3.files.universalopener cimport uopen
17 from obitools3.utils cimport tostr
18 from obitools3.dms.obiseq cimport Nuc_Seq
19 from .embl_genbank_features import extractTaxon
20
21 from libc.stdlib cimport free, malloc, realloc
22 from libc.string cimport strcpy, strlen
23
24
25 _featureMatcher = re.compile(b'^FEATURES.+\n(?=ORIGIN(\s*))',re.DOTALL + re.M)
26
27 _headerMatcher = re.compile(b'^LOCUS.+(?=\nFEATURES)', re.DOTALL + re.M)
28 _seqMatcher = re.compile(b'^ORIGIN.+(?=//\n)', re.DOTALL + re.M)
29 _cleanSeq1 = re.compile(b'ORIGIN(\s*)\n')
30 _cleanSeq2 = re.compile(b'[ \n0-9]+')
31 _acMatcher = re.compile(b'(?<=^ACCESSION ).+',re.M)
32 _deMatcher = re.compile(b'(?<=^DEFINITION ).+\n( .+\n)*',re.M)
33 _cleanDe = re.compile(b'\n *')
34
35
36 def genbankParser(bytes text):
37
38 cdef Nuc_Seq seq
39
40 try:
41 header = _headerMatcher.search(text).group()
42
43 ft = _featureMatcher.search(text).group()
44
45 s = _seqMatcher.search(text).group()
46 s = _cleanSeq1.sub(b'', s)
47 s = _cleanSeq2.sub(b'', s)
48
49 acs = _acMatcher.search(text).group()
50 acs = acs.split()
51 ac = acs[0]
52 acs = acs[1:]
53
54 de = _deMatcher.search(header).group()
55 de = _cleanDe.sub(b' ',de).strip().strip(b'.')
56
57 tags = {}
58 extractTaxon(ft, tags)
59
60 seq = Nuc_Seq(ac,
61 s,
62 definition=de,
63 quality=None,
64 offset=-1,
65 tags=tags)
66
67 except Exception as e:
68 print("\nCould not import sequence id:", text.split()[1], "(error raised:", e, ")")
69 # Do not raise any Exception if you need the possibility to resume the generator
70 # (Python generators can't resume after any exception is raised)
71 return None
72
73 return seq
74
75
76 def genbankIterator_file(lineiterator,
77 int skip=0,
78 only=None,
79 firstline=None,
80 int buffersize=100000000
81 ):
82 cdef int lines_to_skip, ionly, read
83 cdef Nuc_Seq seq
84 cdef char* entry = NULL
85 cdef size_t entry_buffer_size
86 cdef int entry_len
87 cdef int line_len
88
89 entry_buffer_size = 2048
90
91 entry = <char*> malloc(entry_buffer_size*sizeof(char))
92
93 if only is None:
94 ionly = -1
95 else:
96 ionly = int(only)
97
98 if isinstance(lineiterator, (str, bytes)):
99 lineiterator=uopen(lineiterator)
100 if isinstance(lineiterator, LineBuffer):
101 iterator = iter(lineiterator)
102 else:
103 if hasattr(lineiterator, "readlines"):
104 iterator = iter(LineBuffer(lineiterator, buffersize))
105 elif hasattr(lineiterator, '__next__'):
106 iterator = lineiterator
107 else:
108 raise Exception("Invalid line iterator")
109
110 skipped = 0
111 read = 0
112
113 if firstline is None:
114 line = next(iterator)
115 else:
116 line = firstline
117
118 while True:
119
120 if ionly >= 0 and read >= ionly-1:
121 break
122
123 while skipped < skip:
124 line = next(iterator)
125 try:
126 while line[:2] != b'//':
127 line = next(iterator)
128 line = next(iterator)
129 except StopIteration:
130 break
131 skipped += 1
132
133 try:
134 entry_len = 0
135 while line[:2] != b'//':
136 line_len = strlen(line)
137 while (entry_len + line_len) >= entry_buffer_size:
138 entry_buffer_size*=2
139 entry = <char*>realloc(entry, entry_buffer_size)
140 strcpy(entry+entry_len, line)
141 entry_len+=line_len
142 line = next(iterator)
143 # Add last line too because need the // flag to parse
144 line_len = strlen(line)
145 while (entry_len + line_len) >= entry_buffer_size:
146 entry_buffer_size*=2
147 entry = <char*>realloc(entry, entry_buffer_size)
148 strcpy(entry+entry_len, line)
149 line = next(iterator)
150 except StopIteration:
151 break
152
153 seq = genbankParser(entry)
154
155 yield seq
156 read+=1
157
158 # Last sequence if not empty lines
159 if entry.strip():
160 seq = genbankParser(entry)
161 yield seq
162
163 free(entry)
164
165
166 def genbankIterator_dir(dir_path,
167 int skip=0,
168 only=None,
169 firstline=None,
170 int buffersize=100000000
171 ):
172 path = dir_path
173 read = 0
174 read_files = 0
175 files = [filename for filename in glob.glob(os.path.join(path, b'*.gbff*'))]
176 files.extend([filename for filename in glob.glob(os.path.join(path, b'*.seq*'))]) # new genbank extension
177 files = list(set(files))
178 for filename in files:
179 if read==only:
180 return
181 print("Parsing file %s (%d/%d)" % (tostr(filename), read_files+1, len(files)))
182 f = uopen(filename)
183 if only is not None:
184 only_f = only-read
185 else:
186 only_f = None
187 for seq in genbankIterator_file(f, skip=skip, only=only_f, buffersize=buffersize):
188 yield seq
189 read+=1
190 read_files+=1
191
192
193 def genbankIterator(obj,
194 int skip=0,
195 only=None,
196 firstline=None,
197 int buffersize=100000000
198 ):
199 if type(obj) == bytes or type(obj) == str :
200 return genbankIterator_dir(obj, skip=skip, only=only, firstline=firstline, buffersize=buffersize)
201 else:
202 return genbankIterator_file(obj, skip=skip, only=only, firstline=firstline, buffersize=buffersize)
203
204