Debian Codegraph @

resources/scraper.js

raw
1 /* vim:fileencoding=utf-8
2 *
3 * Copyright (C) 2022 Kovid Goyal <kovid at kovidgoyal.net>
4 *
5 * Distributed under terms of the GPLv3 license
6 */
7
8
9 (function() {
10 "use strict";
11
12 var messages = [];
13 var live_requests = {};
14
15 function send_msg(data) {
16 const token = 'TOKEN';
17 const msg = token + ' ' + JSON.stringify(data);
18 console.log(msg);
19 }
20
21 function base64_to_bytes(base64) {
22 const bin_string = atob(base64);
23 return Uint8Array.from(bin_string, (m) => m.codePointAt(0));
24 }
25
26 function notify_that_messages_are_available() {
27 send_msg({type: 'messages_available', count: messages.length});
28 }
29
30 function notify(type, msg) {
31 msg.type = type;
32 messages.push(msg);
33 notify_that_messages_are_available();
34 }
35
36 async function* stream_to_async_iterable(stream) {
37 const reader = stream.getReader()
38 try {
39 while (true) {
40 const {done, value} = await reader.read()
41 if (done) return
42 yield value
43 }
44 } finally {
45 reader.releaseLock()
46 }
47 }
48
49 async function download(req, data) {
50 try {
51 const controller = new AbortController();
52 live_requests[req.id] = controller;
53 var fetch_options = {
54 method: req.method.toUpperCase(),
55 signal: controller.signal,
56 };
57 if (data && data.length > 0) fetch_options.body = base64_to_bytes(data);
58 if (req.headers) {
59 const headers = new Headers();
60 for (const p of req.headers) {
61 headers.append(p[0], p[1]);
62 }
63 fetch_options.headers = headers;
64 }
65 const response = await fetch(req.url, fetch_options);
66 var headers = [];
67 for (const pair of response.headers) {
68 headers.push(pair);
69 }
70 notify('metadata_received', {
71 status_code: response.status, status_msg: response.statusText,
72 url: response.url, headers: headers, response_type: response.type,
73 })
74 for await (const chunk of stream_to_async_iterable(response.body)) {
75 notify('chunk_received', {chunk: chunk})
76 }
77 delete live_requests[req.id];
78 notify('finished', {})
79 } catch (error) {
80 notify('error', {error: error.message});
81 }
82 }
83
84 function abort_download(req_id) {
85 var controller = live_requests[req_id];
86 if (controller) {
87 controller.abort();
88 return true;
89 }
90 return false;
91 }
92
93 function get_messages() {
94 var ans = messages;
95 messages = [];
96 return ans;
97 }
98
99 window.get_messages = get_messages;
100 window.abort_download = abort_download;
101 const payload = JSON.parse(document.getElementById('payload').textContent);
102 download(payload.req, payload.data);
103 })();