| 1 |
|
| 2 |
|
| 3 |
class BuildSplitsets |
| 4 |
|
| 5 |
@@help = [] |
| 6 |
@@help << "build_splitset.rb [options]" |
| 7 |
@@help << "Options:" |
| 8 |
@@help << "-i \t inputfile with sequences" |
| 9 |
@@help << "-o \t path where to put the generated split sets training/testing(crossvalidation) and hold-out (if not existent it will be generated)" |
| 10 |
@@help << "-f \t x fold crossvalidation" |
| 11 |
@@help << "-s \t how much percent of the sequences shoud be used for the training/testing set. The remaining sequences are used for the hold-out set" |
| 12 |
@@help << "-p \t plot the distributions of the GO-Numbers" |
| 13 |
@@help << "-h(elp) print this help" |
| 14 |
def initialize(opts) |
| 15 |
@xfold = 0 |
| 16 |
@split = 0 |
| 17 |
@seqs = [] |
| 18 |
@seqid = 0 |
| 19 |
@plotfile = "" |
| 20 |
@plot =false |
| 21 |
@outpath = "" |
| 22 |
@cv_files = [] |
| 23 |
i = 0 |
| 24 |
|
| 25 |
# parse options |
| 26 |
while i < opts.size |
| 27 |
if opts[i].eql?("-f") |
| 28 |
if opts[i+1] =~ /\d+/ |
| 29 |
@xfold = opts[i+1].to_i |
| 30 |
i+=1 |
| 31 |
else |
| 32 |
raise "Option -f needs an integer parameter!" |
| 33 |
end |
| 34 |
elsif opts[i].eql?("-s") |
| 35 |
if opts[i+1] =~ /\d+/ |
| 36 |
@split = opts[i+1].to_i |
| 37 |
i+=1 |
| 38 |
else |
| 39 |
raise "Option -s needs an integer parameter!" |
| 40 |
end |
| 41 |
elsif opts[i].eql?("-i") |
| 42 |
if File.exists?(opts[i+1]) |
| 43 |
file = File.open(opts[i+1],'rb') |
| 44 |
lines = file.readlines |
| 45 |
parseSequences(lines) |
| 46 |
if opts[i+1] =~ /(\d+)\.f/ # for the naming of the plot files |
| 47 |
@seqid = $1 |
| 48 |
end |
| 49 |
i+=1 |
| 50 |
else |
| 51 |
raise "File #{opts[i+1]} does not exist!" |
| 52 |
end |
| 53 |
elsif opts[i].eql?("-p") |
| 54 |
@plot = true |
| 55 |
elsif opts[i].eql?("-o") |
| 56 |
if !(opts[i+1].length.nil?) |
| 57 |
@outpath = File.expand_path(opts[i+1])+"/" |
| 58 |
i+=1 |
| 59 |
else |
| 60 |
raise "-o needs a directoryname" |
| 61 |
end |
| 62 |
elsif opts[i].eql?("-h") || opts[i].eql?("-help") |
| 63 |
puts @@help |
| 64 |
exit(0) |
| 65 |
else |
| 66 |
raise "Unknown option #{opts[i]}" |
| 67 |
end |
| 68 |
i+=1 |
| 69 |
end |
| 70 |
|
| 71 |
if @outpath.eql?("") |
| 72 |
puts "ERROR: option -o has to be set!" |
| 73 |
puts @@help |
| 74 |
exit(0) |
| 75 |
elsif @xfold == 0 |
| 76 |
puts "ERROR: option -f has to be set and to be greater than 0!" |
| 77 |
puts @@help |
| 78 |
exit(0) |
| 79 |
elsif @split == 0 |
| 80 |
raise "ERROR: option -s has to be set and to be greater than 0!" |
| 81 |
puts @@help |
| 82 |
exit(0) |
| 83 |
elsif @seqs.size < 1 |
| 84 |
raise "ERROR: option -i has to be set!" |
| 85 |
puts @@help |
| 86 |
exit(0) |
| 87 |
end |
| 88 |
#@outpath = @outpath+@seqid+"/" # uncomment for xargs command |
| 89 |
system "mkdir #{@outpath}" |
| 90 |
#get go stats and make tabular file for plotting if -p option is set |
| 91 |
if @plot |
| 92 |
makePlots |
| 93 |
end |
| 94 |
|
| 95 |
#randomize sequences |
| 96 |
random = @seqs.sort_by{rand(@seqs.size)} |
| 97 |
@seqs = random |
| 98 |
|
| 99 |
#split in test/training and hold-out set |
| 100 |
buildSplitsets |
| 101 |
|
| 102 |
#build combinatoric file for crossvalidation |
| 103 |
buildCombinatoricFile |
| 104 |
end |
| 105 |
|
| 106 |
|
| 107 |
def parseSequences(lines) |
| 108 |
i = 0 |
| 109 |
s = "" |
| 110 |
while i < lines.size |
| 111 |
if lines[i] =~ /^>/ |
| 112 |
s = lines[i] |
| 113 |
i+=1 |
| 114 |
while !(lines[i] =~ /^>/) && !(lines[i].nil?) |
| 115 |
s+=lines[i] |
| 116 |
i+=1 |
| 117 |
end |
| 118 |
@seqs << s |
| 119 |
s = "" |
| 120 |
else |
| 121 |
raise lines[i] |
| 122 |
end |
| 123 |
end |
| 124 |
end |
| 125 |
|
| 126 |
def makePlots |
| 127 |
|
| 128 |
#first plot, x-GO:Number, y-frequency |
| 129 |
go_counter = Hash.new |
| 130 |
z = 0 |
| 131 |
@seqs.each do |s| |
| 132 |
if s =~ /((GO:\d+,{0,1})+)/ |
| 133 |
z = z+1 |
| 134 |
gos = $1 |
| 135 |
go_numbers = gos.split(",") |
| 136 |
go_numbers.each do |g| |
| 137 |
if go_counter[g].nil? |
| 138 |
go_counter[g] = 1 |
| 139 |
else |
| 140 |
go_counter[g]+=1 |
| 141 |
end |
| 142 |
end |
| 143 |
else |
| 144 |
raise s |
| 145 |
end |
| 146 |
end |
| 147 |
plotfile = @outpath+"plotfile_1" |
| 148 |
p_out = File.open(plotfile,'wb') |
| 149 |
go_counter.each_key do |k| |
| 150 |
p_out.write k+"\t"+go_counter[k].to_s+"\n" |
| 151 |
end |
| 152 |
p_out.close |
| 153 |
r_file = File.open(plotfile+".R", 'w') |
| 154 |
r_file.write("x <- read.table(\"#{plotfile}\", header=F)\n") |
| 155 |
r_file.write("plot(x[[1]],x[[2]], main=\"GO-Number frequency\", xlab=\"GO-Number\", ylab=\"frequency\", type=\"l\")") |
| 156 |
r_file.close |
| 157 |
system "R -f #{plotfile}.R " |
| 158 |
system "mv ./Rplots.pdf #{@outpath}Rplots#{@seqid}_1.pdf" |
| 159 |
|
| 160 |
|
| 161 |
#second plot, x-frequency, y-amount of x-frequency occurred |
| 162 |
frequency_counter = Hash.new |
| 163 |
go_counter.each_key do |g| |
| 164 |
if frequency_counter[go_counter[g].to_s].nil? |
| 165 |
frequency_counter[go_counter[g].to_s] = 1 |
| 166 |
else |
| 167 |
frequency_counter[go_counter[g].to_s]+=1 |
| 168 |
end |
| 169 |
end |
| 170 |
a = [] |
| 171 |
i = 0 |
| 172 |
frequency_counter.each_key do |k| |
| 173 |
a[k.to_i] = frequency_counter[k] |
| 174 |
end |
| 175 |
while i < a.size |
| 176 |
if a[i].nil? |
| 177 |
a[i] = 0 |
| 178 |
# elsif i < 10 #set the threshold here to remove overrepresentatives in the plot |
| 179 |
# a[i] = 0 |
| 180 |
end |
| 181 |
i+=1 |
| 182 |
end |
| 183 |
plotfile = @outpath+"plotfile_2" |
| 184 |
p_out = File.open(plotfile,'wb') |
| 185 |
a.each_index do |k| |
| 186 |
p_out.write k.to_s+"\t"+a[k].to_s+"\n" |
| 187 |
end |
| 188 |
p_out.close |
| 189 |
r_file = File.open(plotfile+".R", 'w') |
| 190 |
r_file.write("x <- read.table(\"#{plotfile}\", header=F)\n") |
| 191 |
r_file.write("plot(x[[1]],x[[2]], main=\"GO-Number frequency distribution\", xlab=\"frequency\", ylab=\"occurrence\", type=\"l\")") |
| 192 |
r_file.close |
| 193 |
system "R -f #{plotfile}.R" |
| 194 |
system "mv ./Rplots.pdf #{@outpath}Rplots#{@seqid}_2.pdf" |
| 195 |
end |
| 196 |
|
| 197 |
def buildSplitsets |
| 198 |
puts "#{@seqs.size} sequences in sequence file!" |
| 199 |
n = (@seqs.size).to_f * (@split.to_f / 100.0) |
| 200 |
puts "There will be #{n} sequences within the training/testset!" |
| 201 |
n = n.to_i |
| 202 |
i = 0 |
| 203 |
tt_set = [] |
| 204 |
ho_set = [] |
| 205 |
while i < @seqs.size |
| 206 |
if i < n |
| 207 |
tt_set << @seqs[i] |
| 208 |
else |
| 209 |
ho_set << @seqs[i] |
| 210 |
end |
| 211 |
i+=1 |
| 212 |
end |
| 213 |
puts "The training/test set has #{tt_set.size} sequences!" |
| 214 |
puts "The hold-out set has #{ho_set.size} sequences!" |
| 215 |
#write hold-out set into jobfolder/holdoutXX_set.f |
| 216 |
holdout = File.open(@outpath+"holdout#{@seqid}_set.f", 'wb') |
| 217 |
ho_set.each {|seq| holdout.write(seq)} |
| 218 |
holdout.close |
| 219 |
|
| 220 |
#build the @xfold split set files |
| 221 |
n = (tt_set.size).to_f / (@xfold.to_f) |
| 222 |
puts "Each of the #{@xfold} crossvalidation sets will have #{n} sequences!" |
| 223 |
rest = tt_set.size % @xfold.to_f |
| 224 |
puts "#{rest} sequences will be equally distributed among the #{@xfold} sets!" |
| 225 |
n = n.to_i |
| 226 |
cross = [] |
| 227 |
i = 0 |
| 228 |
while i < @xfold |
| 229 |
j = 0 |
| 230 |
cross[i] = [] |
| 231 |
while j < n |
| 232 |
index = j+i*n |
| 233 |
cross[i] << tt_set[index] |
| 234 |
j+=1 |
| 235 |
end |
| 236 |
if rest > 0 |
| 237 |
cross[i] << tt_set[tt_set.size-1-rest] #its better to have balanced sets if the amount of sequences is not dividable by @xfold, |
| 238 |
rest = rest-1 #rather than pushing all of the remaining sequences in the last crossvalidation split set |
| 239 |
end |
| 240 |
i+=1 |
| 241 |
end |
| 242 |
#write all the crossvalidation splitsets in the jobfolder |
| 243 |
i = 0 |
| 244 |
cross.each do |split| |
| 245 |
i+=1 |
| 246 |
puts "Split #{i} has #{split.size} sequences!" |
| 247 |
cv_file = @outpath+"cv_split_#{@seqid}_#{i}.f" |
| 248 |
@cv_files << cv_file |
| 249 |
out = File.open(cv_file,'wb') |
| 250 |
split.each do |seq| |
| 251 |
out.write(seq) |
| 252 |
end |
| 253 |
out.close |
| 254 |
end |
| 255 |
end |
| 256 |
|
| 257 |
def buildCombinatoricFile |
| 258 |
combfile = File.open(@outpath+"crossvalidation_combinatorics.txt", 'w') |
| 259 |
@cv_files.each_index do |i| |
| 260 |
line = "" |
| 261 |
@cv_files.each_index do |j| |
| 262 |
if i != j # file i is added as the last one |
| 263 |
line = line+@cv_files[j]+"\t" |
| 264 |
end |
| 265 |
end |
| 266 |
line = line+@cv_files[i] |
| 267 |
combfile.write line+"\n" |
| 268 |
end |
| 269 |
combfile.close |
| 270 |
end |
| 271 |
|
| 272 |
end |
| 273 |
|
| 274 |
|
| 275 |
BuildSplitsets.new(ARGV) |