Debian Codegraph @

miniget/src/index.ts

raw ◀ ▶
1 Debian patch touch this file
1 import { RequestOptions, IncomingMessage, ClientRequest, default as http } from 'http';
2 import { EventEmitter } from 'events';
3 import https from 'https';
4 import { PassThrough, Transform } from 'stream';
5
6
7 const httpLibs: {
8 [key: string]: {
9 request: (options: RequestOptions | string | URL, callback?: (res: IncomingMessage) => void) => ClientRequest;
10 };
11 } = { 'http:': http, 'https:': https };
12 const redirectStatusCodes = new Set([301, 302, 303, 307, 308]);
13 const retryStatusCodes = new Set([429, 503]);
14
15 // `request`, `response`, `abort`, left out, miniget will emit these.
16 const requestEvents = ['connect', 'continue', 'information', 'socket', 'timeout', 'upgrade'];
17 const responseEvents = ['aborted'];
18
19 namespace Miniget {
20 export interface Options extends RequestOptions {
21 maxRedirects?: number;
22 maxRetries?: number;
23 maxReconnects?: number;
24 backoff?: { inc: number; max: number };
25 highWaterMark?: number;
26 transform?: (parsedUrl: RequestOptions) => RequestOptions;
27 acceptEncoding?: { [key: string]: () => Transform };
28 }
29
30 export interface DefaultOptions extends Options {
31 maxRedirects: number;
32 maxRetries: number;
33 maxReconnects: number;
34 backoff: { inc: number; max: number };
35 }
36
37 export type defaultOptions = Miniget.Options;
38 export type MinigetError = Error;
39
40 export interface Stream extends PassThrough {
41 abort: (err?: Error) => void;
42 aborted: boolean;
43 destroy: (err?: Error) => this;
44 destroyed: boolean;
45 text: () => Promise<string>;
46 on(event: 'reconnect', listener: (attempt: number, err?: Miniget.MinigetError) => void): this;
47 on(event: 'retry', listener: (attempt: number, err?: Miniget.MinigetError) => void): this;
48 on(event: 'redirect', listener: (url: string) => void): this;
49 on(event: string | symbol, listener: (...args: any) => void): this;
50 }
51 }
52
53 Miniget.MinigetError = class MinigetError extends Error {
54 public statusCode?: number;
55 constructor(message: string, statusCode?: number) {
56 super(message);
57 this.statusCode = statusCode;
58 }
59 };
60
61 Miniget.defaultOptions = {
62 maxRedirects: 10,
63 maxRetries: 2,
64 maxReconnects: 0,
65 backoff: { inc: 100, max: 10000 },
66 };
67
68 function Miniget(url: string | URL, options: Miniget.Options = {}): Miniget.Stream {
69 const opts: Miniget.DefaultOptions = Object.assign({}, Miniget.defaultOptions, options);
70 const stream = new PassThrough({ highWaterMark: opts.highWaterMark }) as Miniget.Stream;
71 stream.destroyed = stream.aborted = false;
72 let activeRequest: ClientRequest;
73 let activeResponse: IncomingMessage | null;
74 let activeDecodedStream: Transform | null;
75 let redirects = 0;
76 let retries = 0;
77 let retryTimeout: NodeJS.Timer;
78 let reconnects = 0;
79 let contentLength: number;
80 let acceptRanges = false;
81 let rangeStart = 0, rangeEnd: number;
82 let downloaded = 0;
83
84 // Check if this is a ranged request.
•85 // @ts-ignore
86 if (opts.headers?.Range) {
•87 // @ts-ignore
88 let r = /bytes=(\d+)-(\d+)?/.exec(`${opts.headers.Range}`);
89 if (r) {
90 rangeStart = parseInt(r[1], 10);
91 rangeEnd = parseInt(r[2], 10);
92 }
93 }
94
95 // Add `Accept-Encoding` header.
96 if (opts.acceptEncoding) {
97 opts.headers = Object.assign({
98 'Accept-Encoding': Object.keys(opts.acceptEncoding).join(', '),
99 }, opts.headers);
100 }
101
102 const downloadHasStarted = () => activeDecodedStream && downloaded > 0;
103 const downloadComplete = () => !acceptRanges || downloaded === contentLength;
104
105 const reconnect = (err?: Miniget.MinigetError) => {
106 activeDecodedStream = null;
107 retries = 0;
108 let inc = opts.backoff.inc;
109 let ms = Math.min(inc, opts.backoff.max);
110 retryTimeout = setTimeout(doDownload, ms);
111 stream.emit('reconnect', reconnects, err);
112 };
113
114 const reconnectIfEndedEarly = (err?: Miniget.MinigetError) => {
115 if (options.method !== 'HEAD' && !downloadComplete() && reconnects++ < opts.maxReconnects) {
116 reconnect(err);
117 return true;
118 }
119 return false;
120 };
121
122 interface RetryOptions {
123 err?: Miniget.MinigetError;
124 retryAfter?: number;
125 }
126 const retryRequest = (retryOptions: RetryOptions): boolean => {
127 if (stream.destroyed) { return false; }
128 if (downloadHasStarted()) {
129 return reconnectIfEndedEarly(retryOptions.err);
130 } else if (
131 (!retryOptions.err || retryOptions.err.message === 'ENOTFOUND') &&
132 retries++ < opts.maxRetries) {
133 let ms = retryOptions.retryAfter ||
134 Math.min(retries * opts.backoff.inc, opts.backoff.max);
135 retryTimeout = setTimeout(doDownload, ms);
136 stream.emit('retry', retries, retryOptions.err);
137 return true;
138 }
139 return false;
140 };
141
142 const forwardEvents = (ee: EventEmitter, events: string[]) => {
143 for (let event of events) {
144 ee.on(event, stream.emit.bind(stream, event));
145 }
146 };
147
148 const doDownload = () => {
149 let parsed: RequestOptions = {}, httpLib;
150 try {
151 let urlObj = typeof url === 'string' ? new URL(url) : url;
152 parsed = Object.assign({}, {
153 host: urlObj.host,
154 hostname: urlObj.hostname,
155 path: urlObj.pathname + urlObj.search + urlObj.hash,
156 port: urlObj.port,
157 protocol: urlObj.protocol,
158 });
159 if (urlObj.username) {
160 parsed.auth = `${urlObj.username}:${urlObj.password}`;
161 }
162 httpLib = httpLibs[String(parsed.protocol)];
163 } catch (err) {
164 // Let the error be caught by the if statement below.
165 }
166 if (!httpLib) {
167 stream.emit('error', new Miniget.MinigetError(`Invalid URL: ${url}`));
168 return;
169 }
170
171 Object.assign(parsed, opts);
172 if (acceptRanges && downloaded > 0) {
173 let start = downloaded + rangeStart;
174 let end = rangeEnd || '';
175 parsed.headers = Object.assign({}, parsed.headers, {
176 Range: `bytes=${start}-${end}`,
177 });
178 }
179
180 if (opts.transform) {
181 try {
182 parsed = opts.transform(parsed);
183 } catch (err) {
184 stream.emit('error', err);
185 return;
186 }
187 if (!parsed || parsed.protocol) {
188 httpLib = httpLibs[String(parsed?.protocol)];
189 if (!httpLib) {
190 stream.emit('error', new Miniget.MinigetError('Invalid URL object from `transform` function'));
191 return;
192 }
193 }
194 }
195
196 const onError = (err: Miniget.MinigetError): void => {
197 if (stream.destroyed || stream.readableEnded) { return; }
198 cleanup();
199 if (!retryRequest({ err })) {
200 stream.emit('error', err);
201 } else {
202 activeRequest.removeListener('close', onRequestClose);
203 }
204 };
205
206 const onRequestClose = () => {
207 cleanup();
208 retryRequest({});
209 };
210
211 const cleanup = () => {
212 activeRequest.removeListener('close', onRequestClose);
213 activeResponse?.removeListener('data', onData);
214 activeDecodedStream?.removeListener('end', onEnd);
215 };
216
217 const onData = (chunk: Buffer) => { downloaded += chunk.length; };
218
219 const onEnd = () => {
220 cleanup();
221 if (!reconnectIfEndedEarly()) {
222 stream.end();
223 }
224 };
225
226 activeRequest = httpLib.request(parsed, (res: IncomingMessage) => {
227 // Needed for node v10, v12.
228 // istanbul ignore next
229 if (stream.destroyed) { return; }
230 if (redirectStatusCodes.has(res.statusCode as number)) {
231 if (redirects++ >= opts.maxRedirects) {
232 stream.emit('error', new Miniget.MinigetError('Too many redirects'));
233 } else {
234 if (res.headers.location) {
235 url = res.headers.location;
236 } else {
237 let err = new Miniget.MinigetError('Redirect status code given with no location', res.statusCode);
238 stream.emit('error', err);
239 cleanup();
240 return;
241 }
242 setTimeout(doDownload, parseInt(res.headers['retry-after'] || '0', 10) * 1000);
243 stream.emit('redirect', url);
244 }
245 cleanup();
246 return;
247
248 // Check for rate limiting.
249 } else if (retryStatusCodes.has(res.statusCode as number)) {
250 if (!retryRequest({ retryAfter: parseInt(res.headers['retry-after'] || '0', 10) })) {
251 let err = new Miniget.MinigetError(`Status code: ${res.statusCode}`, res.statusCode);
252 stream.emit('error', err);
253 }
254 cleanup();
255 return;
256 } else if (res.statusCode && (res.statusCode < 200 || res.statusCode >= 400)) {
257 let err = new Miniget.MinigetError(`Status code: ${res.statusCode}`, res.statusCode);
258 if (res.statusCode >= 500) {
259 onError(err);
260 } else {
261 stream.emit('error', err);
262 }
263 cleanup();
264 return;
265 }
266
267 activeDecodedStream = res as unknown as Transform;
268 if (opts.acceptEncoding && res.headers['content-encoding']) {
269 for (let enc of res.headers['content-encoding'].split(', ').reverse()) {
270 let fn = opts.acceptEncoding[enc];
271 if (fn) {
272 activeDecodedStream = activeDecodedStream.pipe(fn());
273 activeDecodedStream.on('error', onError);
274 }
275 }
276 }
277 if (!contentLength) {
278 contentLength = parseInt(`${res.headers['content-length']}`, 10);
279 acceptRanges = res.headers['accept-ranges'] === 'bytes' &&
280 contentLength > 0 && opts.maxReconnects > 0;
281 }
282 res.on('data', onData);
283 activeDecodedStream.on('end', onEnd);
284 activeDecodedStream.pipe(stream, { end: !acceptRanges });
285 activeResponse = res;
286 stream.emit('response', res);
287 res.on('error', onError);
288 forwardEvents(res, responseEvents);
289 });
290 activeRequest.on('error', onError);
291 activeRequest.on('close', onRequestClose);
292 forwardEvents(activeRequest, requestEvents);
293 if (stream.destroyed) {
294 streamDestroy(...destroyArgs);
295 }
296 stream.emit('request', activeRequest);
297 activeRequest.end();
298 };
299
300 stream.abort = (err?: Error) => {
301 console.warn('`MinigetStream#abort()` has been deprecated in favor of `MinigetStream#destroy()`');
302 stream.aborted = true;
303 stream.emit('abort');
304 stream.destroy(err);
305 };
306
307 let destroyArgs: any[] = [];
308 const streamDestroy = (err?: Error) => {
309 activeRequest.destroy(err);
310 activeDecodedStream?.unpipe(stream);
311 activeDecodedStream?.destroy();
•312 // @ts-ignore
313 clearTimeout(retryTimeout);
314 };
315
316 stream._destroy = (...args: any[]) => {
317 stream.destroyed = true;
318 if (activeRequest) {
319 streamDestroy(...args);
320 } else {
321 destroyArgs = args;
322 }
323 };
324
325 stream.text = () => new Promise((resolve, reject) => {
326 let body = '';
327 stream.setEncoding('utf8');
328 stream.on('data', chunk => body += chunk);
329 stream.on('end', () => resolve(body));
330 stream.on('error', reject);
331 });
332
333 process.nextTick(doDownload);
334 return stream;
335 }
336
337 export = Miniget;