Debian Codegraph @

miniget/src/index.ts

raw ◀ ▶
Source reconstructed at patch state 0 of 1 — not the installed (fully patched) source.
1 Debian patch touch this file
1 import { RequestOptions, IncomingMessage, ClientRequest, default as http } from 'http';
2 import { EventEmitter } from 'events';
3 import https from 'https';
4 import { PassThrough, Transform } from 'stream';
5
6
7 const httpLibs: {
8 [key: string]: {
9 request: (options: RequestOptions | string | URL, callback?: (res: IncomingMessage) => void) => ClientRequest;
10 };
11 } = { 'http:': http, 'https:': https };
12 const redirectStatusCodes = new Set([301, 302, 303, 307, 308]);
13 const retryStatusCodes = new Set([429, 503]);
14
15 // `request`, `response`, `abort`, left out, miniget will emit these.
16 const requestEvents = ['connect', 'continue', 'information', 'socket', 'timeout', 'upgrade'];
17 const responseEvents = ['aborted'];
18
19 namespace Miniget {
20 export interface Options extends RequestOptions {
21 maxRedirects?: number;
22 maxRetries?: number;
23 maxReconnects?: number;
24 backoff?: { inc: number; max: number };
25 highWaterMark?: number;
26 transform?: (parsedUrl: RequestOptions) => RequestOptions;
27 acceptEncoding?: { [key: string]: () => Transform };
28 }
29
30 export interface DefaultOptions extends Options {
31 maxRedirects: number;
32 maxRetries: number;
33 maxReconnects: number;
34 backoff: { inc: number; max: number };
35 }
36
37 export type defaultOptions = Miniget.Options;
38 export type MinigetError = Error;
39
40 export interface Stream extends PassThrough {
41 abort: (err?: Error) => void;
42 aborted: boolean;
43 destroy: (err?: Error) => this;
44 destroyed: boolean;
45 text: () => Promise<string>;
46 on(event: 'reconnect', listener: (attempt: number, err?: Miniget.MinigetError) => void): this;
47 on(event: 'retry', listener: (attempt: number, err?: Miniget.MinigetError) => void): this;
48 on(event: 'redirect', listener: (url: string) => void): this;
49 on(event: string | symbol, listener: (...args: any) => void): this;
50 }
51 }
52
53 Miniget.MinigetError = class MinigetError extends Error {
54 public statusCode?: number;
55 constructor(message: string, statusCode?: number) {
56 super(message);
57 this.statusCode = statusCode;
58 }
59 };
60
61 Miniget.defaultOptions = {
62 maxRedirects: 10,
63 maxRetries: 2,
64 maxReconnects: 0,
65 backoff: { inc: 100, max: 10000 },
66 };
67
68 function Miniget(url: string | URL, options: Miniget.Options = {}): Miniget.Stream {
69 const opts: Miniget.DefaultOptions = Object.assign({}, Miniget.defaultOptions, options);
70 const stream = new PassThrough({ highWaterMark: opts.highWaterMark }) as Miniget.Stream;
71 stream.destroyed = stream.aborted = false;
72 let activeRequest: ClientRequest;
73 let activeResponse: IncomingMessage | null;
74 let activeDecodedStream: Transform | null;
75 let redirects = 0;
76 let retries = 0;
77 let retryTimeout: NodeJS.Timer;
78 let reconnects = 0;
79 let contentLength: number;
80 let acceptRanges = false;
81 let rangeStart = 0, rangeEnd: number;
82 let downloaded = 0;
83
84 // Check if this is a ranged request.
85 if (opts.headers?.Range) {
86 let r = /bytes=(\d+)-(\d+)?/.exec(`${opts.headers.Range}`);
87 if (r) {
88 rangeStart = parseInt(r[1], 10);
89 rangeEnd = parseInt(r[2], 10);
90 }
91 }
92
93 // Add `Accept-Encoding` header.
94 if (opts.acceptEncoding) {
95 opts.headers = Object.assign({
96 'Accept-Encoding': Object.keys(opts.acceptEncoding).join(', '),
97 }, opts.headers);
98 }
99
100 const downloadHasStarted = () => activeDecodedStream && downloaded > 0;
101 const downloadComplete = () => !acceptRanges || downloaded === contentLength;
102
103 const reconnect = (err?: Miniget.MinigetError) => {
104 activeDecodedStream = null;
105 retries = 0;
106 let inc = opts.backoff.inc;
107 let ms = Math.min(inc, opts.backoff.max);
108 retryTimeout = setTimeout(doDownload, ms);
109 stream.emit('reconnect', reconnects, err);
110 };
111
112 const reconnectIfEndedEarly = (err?: Miniget.MinigetError) => {
113 if (options.method !== 'HEAD' && !downloadComplete() && reconnects++ < opts.maxReconnects) {
114 reconnect(err);
115 return true;
116 }
117 return false;
118 };
119
120 interface RetryOptions {
121 err?: Miniget.MinigetError;
122 retryAfter?: number;
123 }
124 const retryRequest = (retryOptions: RetryOptions): boolean => {
125 if (stream.destroyed) { return false; }
126 if (downloadHasStarted()) {
127 return reconnectIfEndedEarly(retryOptions.err);
128 } else if (
129 (!retryOptions.err || retryOptions.err.message === 'ENOTFOUND') &&
130 retries++ < opts.maxRetries) {
131 let ms = retryOptions.retryAfter ||
132 Math.min(retries * opts.backoff.inc, opts.backoff.max);
133 retryTimeout = setTimeout(doDownload, ms);
134 stream.emit('retry', retries, retryOptions.err);
135 return true;
136 }
137 return false;
138 };
139
140 const forwardEvents = (ee: EventEmitter, events: string[]) => {
141 for (let event of events) {
142 ee.on(event, stream.emit.bind(stream, event));
143 }
144 };
145
146 const doDownload = () => {
147 let parsed: RequestOptions = {}, httpLib;
148 try {
149 let urlObj = typeof url === 'string' ? new URL(url) : url;
150 parsed = Object.assign({}, {
151 host: urlObj.host,
152 hostname: urlObj.hostname,
153 path: urlObj.pathname + urlObj.search + urlObj.hash,
154 port: urlObj.port,
155 protocol: urlObj.protocol,
156 });
157 if (urlObj.username) {
158 parsed.auth = `${urlObj.username}:${urlObj.password}`;
159 }
160 httpLib = httpLibs[String(parsed.protocol)];
161 } catch (err) {
162 // Let the error be caught by the if statement below.
163 }
164 if (!httpLib) {
165 stream.emit('error', new Miniget.MinigetError(`Invalid URL: ${url}`));
166 return;
167 }
168
169 Object.assign(parsed, opts);
170 if (acceptRanges && downloaded > 0) {
171 let start = downloaded + rangeStart;
172 let end = rangeEnd || '';
173 parsed.headers = Object.assign({}, parsed.headers, {
174 Range: `bytes=${start}-${end}`,
175 });
176 }
177
178 if (opts.transform) {
179 try {
180 parsed = opts.transform(parsed);
181 } catch (err) {
182 stream.emit('error', err);
183 return;
184 }
185 if (!parsed || parsed.protocol) {
186 httpLib = httpLibs[String(parsed?.protocol)];
187 if (!httpLib) {
188 stream.emit('error', new Miniget.MinigetError('Invalid URL object from `transform` function'));
189 return;
190 }
191 }
192 }
193
194 const onError = (err: Miniget.MinigetError): void => {
195 if (stream.destroyed || stream.readableEnded) { return; }
196 cleanup();
197 if (!retryRequest({ err })) {
198 stream.emit('error', err);
199 } else {
200 activeRequest.removeListener('close', onRequestClose);
201 }
202 };
203
204 const onRequestClose = () => {
205 cleanup();
206 retryRequest({});
207 };
208
209 const cleanup = () => {
210 activeRequest.removeListener('close', onRequestClose);
211 activeResponse?.removeListener('data', onData);
212 activeDecodedStream?.removeListener('end', onEnd);
213 };
214
215 const onData = (chunk: Buffer) => { downloaded += chunk.length; };
216
217 const onEnd = () => {
218 cleanup();
219 if (!reconnectIfEndedEarly()) {
220 stream.end();
221 }
222 };
223
224 activeRequest = httpLib.request(parsed, (res: IncomingMessage) => {
225 // Needed for node v10, v12.
226 // istanbul ignore next
227 if (stream.destroyed) { return; }
228 if (redirectStatusCodes.has(res.statusCode as number)) {
229 if (redirects++ >= opts.maxRedirects) {
230 stream.emit('error', new Miniget.MinigetError('Too many redirects'));
231 } else {
232 if (res.headers.location) {
233 url = res.headers.location;
234 } else {
235 let err = new Miniget.MinigetError('Redirect status code given with no location', res.statusCode);
236 stream.emit('error', err);
237 cleanup();
238 return;
239 }
240 setTimeout(doDownload, parseInt(res.headers['retry-after'] || '0', 10) * 1000);
241 stream.emit('redirect', url);
242 }
243 cleanup();
244 return;
245
246 // Check for rate limiting.
247 } else if (retryStatusCodes.has(res.statusCode as number)) {
248 if (!retryRequest({ retryAfter: parseInt(res.headers['retry-after'] || '0', 10) })) {
249 let err = new Miniget.MinigetError(`Status code: ${res.statusCode}`, res.statusCode);
250 stream.emit('error', err);
251 }
252 cleanup();
253 return;
254 } else if (res.statusCode && (res.statusCode < 200 || res.statusCode >= 400)) {
255 let err = new Miniget.MinigetError(`Status code: ${res.statusCode}`, res.statusCode);
256 if (res.statusCode >= 500) {
257 onError(err);
258 } else {
259 stream.emit('error', err);
260 }
261 cleanup();
262 return;
263 }
264
265 activeDecodedStream = res as unknown as Transform;
266 if (opts.acceptEncoding && res.headers['content-encoding']) {
267 for (let enc of res.headers['content-encoding'].split(', ').reverse()) {
268 let fn = opts.acceptEncoding[enc];
269 if (fn) {
270 activeDecodedStream = activeDecodedStream.pipe(fn());
271 activeDecodedStream.on('error', onError);
272 }
273 }
274 }
275 if (!contentLength) {
276 contentLength = parseInt(`${res.headers['content-length']}`, 10);
277 acceptRanges = res.headers['accept-ranges'] === 'bytes' &&
278 contentLength > 0 && opts.maxReconnects > 0;
279 }
280 res.on('data', onData);
281 activeDecodedStream.on('end', onEnd);
282 activeDecodedStream.pipe(stream, { end: !acceptRanges });
283 activeResponse = res;
284 stream.emit('response', res);
285 res.on('error', onError);
286 forwardEvents(res, responseEvents);
287 });
288 activeRequest.on('error', onError);
289 activeRequest.on('close', onRequestClose);
290 forwardEvents(activeRequest, requestEvents);
291 if (stream.destroyed) {
292 streamDestroy(...destroyArgs);
293 }
294 stream.emit('request', activeRequest);
295 activeRequest.end();
296 };
297
298 stream.abort = (err?: Error) => {
299 console.warn('`MinigetStream#abort()` has been deprecated in favor of `MinigetStream#destroy()`');
300 stream.aborted = true;
301 stream.emit('abort');
302 stream.destroy(err);
303 };
304
305 let destroyArgs: any[] = [];
306 const streamDestroy = (err?: Error) => {
307 activeRequest.destroy(err);
308 activeDecodedStream?.unpipe(stream);
309 activeDecodedStream?.destroy();
310 clearTimeout(retryTimeout);
311 };
312
313 stream._destroy = (...args: any[]) => {
314 stream.destroyed = true;
315 if (activeRequest) {
316 streamDestroy(...args);
317 } else {
318 destroyArgs = args;
319 }
320 };
321
322 stream.text = () => new Promise((resolve, reject) => {
323 let body = '';
324 stream.setEncoding('utf8');
325 stream.on('data', chunk => body += chunk);
326 stream.on('end', () => resolve(body));
327 stream.on('error', reject);
328 });
329
330 process.nextTick(doDownload);
331 return stream;
332 }
333
334 export = Miniget;