Debian Codegraph @

src/downloadmanagers/fancy.cpp

raw
1 // For all support, instructions and copyright go to:
2 // http://e2guardian.org/
3 // Released under the GPL v2, with the OpenSSL exception described in the README file.
4
5 // INCLUDES
6 #ifdef HAVE_CONFIG_H
7 #include "e2config.h"
8 #endif
9
10 #include "../DownloadManager.hpp"
11 #include "../OptionContainer.hpp"
12 #include "../HTMLTemplate.hpp"
13 #include "../ConnectionHandler.hpp"
14 #include "../Logger.hpp"
15
16 #include <string.h>
17 #include <sys/time.h>
18 #include <sys/types.h>
19 #include <unistd.h>
20 #include <iostream>
21 #include <cmath>
22 #include <cstdio>
23
24 // GLOBALS
25
26 extern OptionContainer o;
27
28 // DECLARATIONS
29
30 class fancydm : public DMPlugin
31 {
32 public:
33 fancydm(ConfigVar &definition)
34 : DMPlugin(definition), upperlimit(0), toobig_unscanned(false), toobig_notdownloaded(false){};
35 int in(DataBuffer *d, Socket *sock, Socket *peersock, HTTPHeader *requestheader,
36 HTTPHeader *docheader, bool wantall, int *headersent, bool *toobig);
37
38 int init(void *args);
39
40 bool sendLink(Socket &peersock, String &linkurl, String &prettyurl);
41
42 private:
43 // customisable fancy DM template
44 HTMLTemplate progresspage;
45
46 // upper (non-scanned) file download size limit
47 // part of "multi-stage" downloads for when content length is unknown:
48 // - start downloading & hope for the best
49 // - if too large, warn the user, but keep downloading (won't be scanned)
50 // - if larger than upper limit also, kill download.
51 off_t upperlimit;
52
53 // was file too large to be scanned?
54 bool toobig_unscanned;
55
56 // was file so large as to not even be downloaded?
57 bool toobig_notdownloaded;
58
59 // format an integer in seconds as hours:minutes:seconds
60 std::string timestring(const int seconds);
61 // format a number of bytes as a number of Gb/Mb/Kb
62 String bytestring(const off_t bytes);
63 };
64
65 // IMPLEMENTATION
66
67 // class factory code *MUST* be included in every plugin
68
69 DMPlugin *fancydmcreate(ConfigVar &definition)
70 {
71 DEBUG_trace("Creating fancy DM");
72 return new fancydm(definition);
73 }
74
75 // end of Class factory
76
77 // initialisation - load the template file
78 int fancydm::init(void *args)
79 {
80 // call inherited init
81 DMPlugin::init(args);
82
83 StoryBoardOptions::SB_entry_map sen;
84 sen.entry_function = cv["story_function"];
85 if (sen.entry_function.length() > 0) {
86 sen.entry_id = ENT_STORYB_DM_FANCY;
87 story_entry = sen.entry_id;
88 o.story.dm_entry_dq.push_back(sen);
89 } else {
90 E2LOGGER_error("No story_function defined in fancy DM plugin config");
91 return -1;
92 }
93
94 // read in absolute max download limit
95 upperlimit = cv["maxdownloadsize"].toOffset() * 1024;
96 if (upperlimit <= o.content.max_content_filecache_scan_size)
97 upperlimit = 0;
98
99 DEBUG_dwload("Upper download limit: ", upperlimit);
100
101 String fname(cv["template"]);
102 if (fname.length() > 0) {
103 // read the template file, and return OK on success, error on failure.
104 fname = o.config.languagepath + fname;
105 return progresspage.readTemplateFile(fname.toCharArray(), "-FILENAME-|-FILESIZE-|-SERVERIP-") ? 0 : -1;
106 } else {
107 // eek! there's no template option in our config.
108 E2LOGGER_error("Template not specified for fancy download manager");
109 return -2;
110 }
111 }
112
113 // call template's downloadlink JavaScript function
114 bool fancydm::sendLink(Socket &peersock, String &linkurl, String &prettyurl)
115 {
116 String mess("<script language='javascript'>\n<!--\ndownloadlink(\"" + linkurl + "\",\"" + prettyurl
117 + "\"," + (toobig_notdownloaded ? "2" : (toobig_unscanned ? "1" : "0"))
118 + ");\n//-->\n</script>\n");
119 peersock.writeString(mess.toCharArray());
120 // send text-only version for non-JS-enabled browsers
121 // 1220 "Scan complete.</p><p>Click here to download: "
122 // 1221 "Download complete; file not scanned.</p><p>Click here to download: "
123 // 1222 "File too large to cache.</p><p>Click here to re-download, bypassing virus scanning: "
124 mess = "<noscript><p>";
125 mess += o.language_list.getTranslation(toobig_notdownloaded ? 1222 : (toobig_unscanned ? 1221 : 1220));
126 mess += "<a href=\"" + linkurl + "\">" + prettyurl + "</a></p></noscript>";
127 peersock.writeString(mess.toCharArray());
128 peersock.writeString("<!-- force flush -->\r\n");
129 if(!peersock.writeString("</body></html>\n")) return false;
130 if (toobig_notdownloaded) {
131 // add URL to clean cache (for all groups)
132 addToClean(prettyurl, o.filter.filter_groups + 1);
133 }
134 return true;
135 }
136
137 // download body for this request
138 int fancydm::in(DataBuffer *d, Socket *sock, Socket *peersock, class HTTPHeader *requestheader, class HTTPHeader *docheader, bool wantall, int *headersent, bool *toobig)
139 {
140
141 //DataBuffer *d = where to stick the data back into
142 //Socket *sock = where to read from
143 //Socket *peersock = browser to send stuff to for keeping it alive
144 //HTTPHeader *docheader = header used for sending first line of reply
145 //HTTPHeader *requestheader = header client used to request
146 //bool wantall = to determine if just content filter or a full scan
147 //int *headersent = to use to send the first line of header if needed
148 // or to mark the header has already been sent
149 //bool *toobig = flag to modify to say if it could not all be downloaded
150
151 DEBUG_trace("Inside fancy download manager plugin");
152
153 //int rc = 0;
154
155 //off_t newsize;
156 off_t expectedsize = docheader->contentLength();
157 d->bytes_toget = expectedsize;
158 off_t bytessec = 0;
159 off_t bytesgot = 0;
160 int percentcomplete = 0;
161 unsigned int eta = 0;
162 int timeelapsed = 0;
163
164 if (!d->icap) {
165 DEBUG_dwload("tranencodeing is ", docheader->transferEncoding());
166 d->chunked = docheader->transferEncoding().contains("chunked");
167 }
168
169 // if using non-persistent connections, some servers will not report
170 // a content-length. in these situations, just download everything.
171 d->geteverything = false;
172 if ((d->bytes_toget < 0) || (d->chunked))
173 d->geteverything = true;
174
175 if (expectedsize < 0)
176 expectedsize = 0;
177
178 bool initialsent = false;
179
180 String message, jsmessage;
181
182 d->swappedtodisk = false;
183 d->doneinitialdelay = false;
184
185 struct timeval starttime;
186 struct timeval themdays;
187 struct timeval nowadays;
188 gettimeofday(&themdays, NULL);
189 gettimeofday(&starttime, NULL);
190
191 toobig_unscanned = false;
192 toobig_notdownloaded = false;
193 bool secondstage = false;
194
195 // buffer size for streaming downloads
196 off_t blocksize = 32768;
197 // set to a sensible minimum
198 if (!wantall && (blocksize > o.content.max_content_filter_size))
199 blocksize = o.content.max_content_filter_size;
200 else if (wantall && (blocksize > o.content.max_content_ramcache_scan_size))
201 blocksize = o.content.max_content_ramcache_scan_size;
202 DEBUG_dwload("blocksize: ", blocksize);
203
204 // determine downloaded filename
205 String filename(requestheader->disposition());
206 if (filename.length() == 0) {
207 filename = requestheader->getUrl();
208 filename = requestheader->decode(filename);
209 if (filename.contains("?"))
210 filename = filename.before("?");
211 while (filename.contains("/"))
212 filename = filename.after("/");
213 }
214
215 while ((bytesgot < expectedsize) || d->geteverything) {
216 // send text header to show status
217 if (o.content.trickle_delay > 0) {
218 gettimeofday(&nowadays, NULL);
219 timeelapsed = nowadays.tv_sec - starttime.tv_sec;
220 if ((!initialsent && timeelapsed > o.content.initial_trickle_delay) ||
221 (initialsent && nowadays.tv_sec - themdays.tv_sec > o.content.trickle_delay)) {
222 initialsent = true;
223 bytessec = bytesgot / timeelapsed;
224 themdays.tv_sec = nowadays.tv_sec;
225 if ((*headersent) < 1) {
226 DEBUG_dwload("sending header for text status");
227 message = "HTTP/1.0 200 OK\nContent-Type: text/html\n\n";
228 // Output initial template
229 std::deque<String>::iterator i = progresspage.html.begin();
230 std::deque<String>::iterator penultimate = progresspage.html.end() - 1;
231 bool newline;
232 while (i != progresspage.html.end()) {
233 newline = false;
234 message = *i;
235 if (message == "-FILENAME-") {
236 message = filename;
237 } else if (message == "-FILESIZE-") {
238 message = String(expectedsize);
239 } else if (message == "-SERVERIP-") {
240 message = peersock->getLocalIP();
241 } else if ((i == penultimate) || ((*(i + 1))[0] != '-')) {
242 newline = true;
243 }
244 peersock->writeString(message.toCharArray());
245 // preserve line breaks from the original template file
246 if (newline)
247 peersock->writeString("\n");
248 i++;
249 }
250 // send please wait message for non-JS-enabled browsers
251 // 1200 "Please wait - downloading to be scanned..."
252 message = "<noscript><p>";
253 message += o.language_list.getTranslation(1200);
254 message += "</p></noscript>\n";
255 peersock->writeString(message.toCharArray());
256 (*headersent) = 2;
257 }
258
259 DEBUG_dwload("trickle delay - sending progress...");
260 message = "Downloading status: ";
261 // Output a call to template's JavaScript progressupdate function
262 jsmessage = "<script language='javascript'>\n<!--\nprogressupdate(" + String(bytesgot) + "," +
263 String(bytessec) + ");\n//-->\n</script>";
264 peersock->writeString(jsmessage.toCharArray());
265 // send text only version for non-JS-enabled browsers.
266 // checkme: translation?
267 if (d->geteverything) {
268 message = "<noscript><p>Time remaining: unknown; "
269 + bytestring(bytessec) + "/s; total downloaded: " + bytestring(bytesgot) +
270 "</p></noscript>\n";
271 } else {
272 percentcomplete = bytesgot / (expectedsize / 100);
273 eta = (expectedsize - bytesgot) / bytessec;
274 message = "<noscript><p>" + String(percentcomplete) + "%, time remaining: " + timestring(eta) + "; "
275 + bytestring(bytessec) + "/s; total downloaded: " + bytestring(bytesgot) +
276 "</p></noscript>\n";
277 }
278 peersock->writeString(message.toCharArray());
279 peersock->writeString("<!-- force flush -->\r\n");
280 }
281 }
282 int read_res;
283 int rc;
284 int bsize = blocksize;
285 if ((!d->geteverything) && (d->bytes_toget < bsize))
286 bsize = d->bytes_toget;
287 DEBUG_dwload("bsize is ", bsize);
288
289 rc = d->readInFromSocket(sock, bsize, wantall, read_res);
290 if (read_res & DB_TOBIG)
291 *toobig = true;
292 if (read_res & DB_TOBIG_SCAN) {
293 // multi-stage download enabled, and we don't know content length
294 toobig_unscanned = true;
295 if (d->geteverything && (upperlimit > 0)) {
296 if ((!secondstage) && initialsent) {
297 secondstage = true;
298 // send download size warning message
299 jsmessage = "<script language='javascript'>\n<!--\ndownloadwarning(" + String(upperlimit) +
300 ");\n//-->\n</script>";
301 peersock->writeString(jsmessage.toCharArray());
302 // text-only version
303 message = "<noscript><p>";
304 // 1201 Warning: file too large to scan. If you suspect that this file is larger than
305 // 1202 , then refresh to download directly.
306 message += o.language_list.getTranslation(1201);
307 message += bytestring(upperlimit);
308 message += o.language_list.getTranslation(1202);
309 message += "</p></noscript>\n";
310 peersock->writeString(message.toCharArray());
311 peersock->writeString("<!-- force flush -->\r\n");
312 // add URL to clean cache (for all groups)
313 // TODO: aah - this will not work without clean cache! Needs a different method????
314 DEBUG_dwload("fancydm: file too big to be scanned, entering second stage of download");
315 }
316
317 // too large to even download, let alone scan
318 if (bytesgot > upperlimit) {
319 DEBUG_dwload("fancydm: file too big to be downloaded, halting second stage of download");
320 toobig_unscanned = false;
321 toobig_notdownloaded = true;
322 break;
323 }
324 } else {
325 // multi-stage download disabled, or we know content length
326 // if swapped to disk and file too large for that too, then give up
327 DEBUG_dwload("fancydm: file too big to be scanned, halting download");
328 toobig_unscanned = false;
329 toobig_notdownloaded = true;
330 break;
331 }
332
333 }
334 if (read_res & DB_TOBIG_FILTER) {
335 // may not need anything here
336 }
337 if (rc <= 0) break;
338 }
339
340 if (initialsent) {
341
342 if (!d->swappedtodisk) { // if we sent textual content then we can't
343 // stream the file to the user so we must save to disk for them
344 // to download by clicking on the magic link
345 // You can get to this point by having a large ram cache, or
346 // slow internet connection with small initial trickle delay.
347 // This should be rare.
348 DEBUG_dwload("swapping to disk");
349 d->tempfilefd = d->getTempFileFD();
350 if (d->tempfilefd < 0) {
351 E2LOGGER_error("error buffering complete to disk so skipping disk buffering");
352 } else {
353 write(d->tempfilefd, d->data, d->buffer_length);
354 d->swappedtodisk = true;
355 d->tempfilesize = d->buffer_length;
356 }
357 }
358
359 if (!(toobig_unscanned || toobig_notdownloaded)) {
360 // Output a call to template's JavaScript nowscanning function
361 peersock->writeString("<script language='javascript'>\n<!--\nnowscanning();\n//-->\n</script>\n");
362 // send text-only version
363 // 1210 "Download Complete. Starting scan..."
364 message = "<noscript><p>";
365 message += o.language_list.getTranslation(1210);
366 message += "</p></noscript>\n";
367 peersock->writeString(message.toCharArray());
368 }
369 // only keep full downloads
370 if (!toobig_notdownloaded)
371 (*d).preservetemp = true;
372 (*d).dontsendbody = true;
373 }
374
375
376 if (!(*toobig) && !d->swappedtodisk) { // won't deflate stuff swapped to disk
377 if (d->decompress.contains("deflate")) {
378 DEBUG_dwload("zlib format");
379 d->zlibinflate(false); // incoming stream was zlib compressed
380 } else if (d->decompress.contains("gzip")) {
381 DEBUG_dwload("gzip format");
382 d->zlibinflate(true); // incoming stream was gzip compressed
383 }
384 }
385 d->bytesalreadysent = 0;
386 return 0;
387 }
388
389 // format an integer in seconds as hours:minutes:seconds
390 std::string fancydm::timestring(const int seconds)
391 {
392 int hours = (int)floor((double)seconds / 3600);
393 int minutes = (int)floor(((double)seconds / 60) - (hours * 3600));
394 int remainingseconds = seconds - (minutes * 60) - (hours * 3600);
395 char result[9];
396 snprintf(result, 9, "%02i:%02i:%02i", hours, minutes, remainingseconds);
397 return result;
398 }
399 // format a number of bytes as a number of Gb/Mb/Kb
400 String fancydm::bytestring(const off_t bytes)
401 {
402 int b = (int)floor((double)bytes / (1024 * 1024 * 1024));
403 String num(b);
404 if (b > 0) {
405 return num += " Gb";
406 }
407 b = (int)floor((double)bytes / (1024 * 1024));
408 num = b;
409 if (b > 0)
410 return num += " Mb";
411 b = (int)floor((double)bytes / 1024);
412 num = b;
413 if (b > 0)
414 return num += " Kb";
415 b = (int)floor((double)bytes);
416 num = b;
417 return num += " bytes";
418 }