| 1 |
import { RequestOptions, IncomingMessage, ClientRequest, default as http } from 'http'; |
| 2 |
import { EventEmitter } from 'events'; |
| 3 |
import https from 'https'; |
| 4 |
import { PassThrough, Transform } from 'stream'; |
| 5 |
|
| 6 |
|
| 7 |
const httpLibs: { |
| 8 |
[key: string]: { |
| 9 |
request: (options: RequestOptions | string | URL, callback?: (res: IncomingMessage) => void) => ClientRequest; |
| 10 |
}; |
| 11 |
} = { 'http:': http, 'https:': https }; |
| 12 |
const redirectStatusCodes = new Set([301, 302, 303, 307, 308]); |
| 13 |
const retryStatusCodes = new Set([429, 503]); |
| 14 |
|
| 15 |
// `request`, `response`, `abort`, left out, miniget will emit these. |
| 16 |
const requestEvents = ['connect', 'continue', 'information', 'socket', 'timeout', 'upgrade']; |
| 17 |
const responseEvents = ['aborted']; |
| 18 |
|
| 19 |
namespace Miniget { |
| 20 |
export interface Options extends RequestOptions { |
| 21 |
maxRedirects?: number; |
| 22 |
maxRetries?: number; |
| 23 |
maxReconnects?: number; |
| 24 |
backoff?: { inc: number; max: number }; |
| 25 |
highWaterMark?: number; |
| 26 |
transform?: (parsedUrl: RequestOptions) => RequestOptions; |
| 27 |
acceptEncoding?: { [key: string]: () => Transform }; |
| 28 |
} |
| 29 |
|
| 30 |
export interface DefaultOptions extends Options { |
| 31 |
maxRedirects: number; |
| 32 |
maxRetries: number; |
| 33 |
maxReconnects: number; |
| 34 |
backoff: { inc: number; max: number }; |
| 35 |
} |
| 36 |
|
| 37 |
export type defaultOptions = Miniget.Options; |
| 38 |
export type MinigetError = Error; |
| 39 |
|
| 40 |
export interface Stream extends PassThrough { |
| 41 |
abort: (err?: Error) => void; |
| 42 |
aborted: boolean; |
| 43 |
destroy: (err?: Error) => this; |
| 44 |
destroyed: boolean; |
| 45 |
text: () => Promise<string>; |
| 46 |
on(event: 'reconnect', listener: (attempt: number, err?: Miniget.MinigetError) => void): this; |
| 47 |
on(event: 'retry', listener: (attempt: number, err?: Miniget.MinigetError) => void): this; |
| 48 |
on(event: 'redirect', listener: (url: string) => void): this; |
| 49 |
on(event: string | symbol, listener: (...args: any) => void): this; |
| 50 |
} |
| 51 |
} |
| 52 |
|
| 53 |
Miniget.MinigetError = class MinigetError extends Error { |
| 54 |
public statusCode?: number; |
| 55 |
constructor(message: string, statusCode?: number) { |
| 56 |
super(message); |
| 57 |
this.statusCode = statusCode; |
| 58 |
} |
| 59 |
}; |
| 60 |
|
| 61 |
Miniget.defaultOptions = { |
| 62 |
maxRedirects: 10, |
| 63 |
maxRetries: 2, |
| 64 |
maxReconnects: 0, |
| 65 |
backoff: { inc: 100, max: 10000 }, |
| 66 |
}; |
| 67 |
|
| 68 |
function Miniget(url: string | URL, options: Miniget.Options = {}): Miniget.Stream { |
| 69 |
const opts: Miniget.DefaultOptions = Object.assign({}, Miniget.defaultOptions, options); |
| 70 |
const stream = new PassThrough({ highWaterMark: opts.highWaterMark }) as Miniget.Stream; |
| 71 |
stream.destroyed = stream.aborted = false; |
| 72 |
let activeRequest: ClientRequest; |
| 73 |
let activeResponse: IncomingMessage | null; |
| 74 |
let activeDecodedStream: Transform | null; |
| 75 |
let redirects = 0; |
| 76 |
let retries = 0; |
| 77 |
let retryTimeout: NodeJS.Timer; |
| 78 |
let reconnects = 0; |
| 79 |
let contentLength: number; |
| 80 |
let acceptRanges = false; |
| 81 |
let rangeStart = 0, rangeEnd: number; |
| 82 |
let downloaded = 0; |
| 83 |
|
| 84 |
// Check if this is a ranged request. |
| 85 |
if (opts.headers?.Range) { |
| 86 |
let r = /bytes=(\d+)-(\d+)?/.exec(`${opts.headers.Range}`); |
| 87 |
if (r) { |
| 88 |
rangeStart = parseInt(r[1], 10); |
| 89 |
rangeEnd = parseInt(r[2], 10); |
| 90 |
} |
| 91 |
} |
| 92 |
|
| 93 |
// Add `Accept-Encoding` header. |
| 94 |
if (opts.acceptEncoding) { |
| 95 |
opts.headers = Object.assign({ |
| 96 |
'Accept-Encoding': Object.keys(opts.acceptEncoding).join(', '), |
| 97 |
}, opts.headers); |
| 98 |
} |
| 99 |
|
| 100 |
const downloadHasStarted = () => activeDecodedStream && downloaded > 0; |
| 101 |
const downloadComplete = () => !acceptRanges || downloaded === contentLength; |
| 102 |
|
| 103 |
const reconnect = (err?: Miniget.MinigetError) => { |
| 104 |
activeDecodedStream = null; |
| 105 |
retries = 0; |
| 106 |
let inc = opts.backoff.inc; |
| 107 |
let ms = Math.min(inc, opts.backoff.max); |
| 108 |
retryTimeout = setTimeout(doDownload, ms); |
| 109 |
stream.emit('reconnect', reconnects, err); |
| 110 |
}; |
| 111 |
|
| 112 |
const reconnectIfEndedEarly = (err?: Miniget.MinigetError) => { |
| 113 |
if (options.method !== 'HEAD' && !downloadComplete() && reconnects++ < opts.maxReconnects) { |
| 114 |
reconnect(err); |
| 115 |
return true; |
| 116 |
} |
| 117 |
return false; |
| 118 |
}; |
| 119 |
|
| 120 |
interface RetryOptions { |
| 121 |
err?: Miniget.MinigetError; |
| 122 |
retryAfter?: number; |
| 123 |
} |
| 124 |
const retryRequest = (retryOptions: RetryOptions): boolean => { |
| 125 |
if (stream.destroyed) { return false; } |
| 126 |
if (downloadHasStarted()) { |
| 127 |
return reconnectIfEndedEarly(retryOptions.err); |
| 128 |
} else if ( |
| 129 |
(!retryOptions.err || retryOptions.err.message === 'ENOTFOUND') && |
| 130 |
retries++ < opts.maxRetries) { |
| 131 |
let ms = retryOptions.retryAfter || |
| 132 |
Math.min(retries * opts.backoff.inc, opts.backoff.max); |
| 133 |
retryTimeout = setTimeout(doDownload, ms); |
| 134 |
stream.emit('retry', retries, retryOptions.err); |
| 135 |
return true; |
| 136 |
} |
| 137 |
return false; |
| 138 |
}; |
| 139 |
|
| 140 |
const forwardEvents = (ee: EventEmitter, events: string[]) => { |
| 141 |
for (let event of events) { |
| 142 |
ee.on(event, stream.emit.bind(stream, event)); |
| 143 |
} |
| 144 |
}; |
| 145 |
|
| 146 |
const doDownload = () => { |
| 147 |
let parsed: RequestOptions = {}, httpLib; |
| 148 |
try { |
| 149 |
let urlObj = typeof url === 'string' ? new URL(url) : url; |
| 150 |
parsed = Object.assign({}, { |
| 151 |
host: urlObj.host, |
| 152 |
hostname: urlObj.hostname, |
| 153 |
path: urlObj.pathname + urlObj.search + urlObj.hash, |
| 154 |
port: urlObj.port, |
| 155 |
protocol: urlObj.protocol, |
| 156 |
}); |
| 157 |
if (urlObj.username) { |
| 158 |
parsed.auth = `${urlObj.username}:${urlObj.password}`; |
| 159 |
} |
| 160 |
httpLib = httpLibs[String(parsed.protocol)]; |
| 161 |
} catch (err) { |
| 162 |
// Let the error be caught by the if statement below. |
| 163 |
} |
| 164 |
if (!httpLib) { |
| 165 |
stream.emit('error', new Miniget.MinigetError(`Invalid URL: ${url}`)); |
| 166 |
return; |
| 167 |
} |
| 168 |
|
| 169 |
Object.assign(parsed, opts); |
| 170 |
if (acceptRanges && downloaded > 0) { |
| 171 |
let start = downloaded + rangeStart; |
| 172 |
let end = rangeEnd || ''; |
| 173 |
parsed.headers = Object.assign({}, parsed.headers, { |
| 174 |
Range: `bytes=${start}-${end}`, |
| 175 |
}); |
| 176 |
} |
| 177 |
|
| 178 |
if (opts.transform) { |
| 179 |
try { |
| 180 |
parsed = opts.transform(parsed); |
| 181 |
} catch (err) { |
| 182 |
stream.emit('error', err); |
| 183 |
return; |
| 184 |
} |
| 185 |
if (!parsed || parsed.protocol) { |
| 186 |
httpLib = httpLibs[String(parsed?.protocol)]; |
| 187 |
if (!httpLib) { |
| 188 |
stream.emit('error', new Miniget.MinigetError('Invalid URL object from `transform` function')); |
| 189 |
return; |
| 190 |
} |
| 191 |
} |
| 192 |
} |
| 193 |
|
| 194 |
const onError = (err: Miniget.MinigetError): void => { |
| 195 |
if (stream.destroyed || stream.readableEnded) { return; } |
| 196 |
cleanup(); |
| 197 |
if (!retryRequest({ err })) { |
| 198 |
stream.emit('error', err); |
| 199 |
} else { |
| 200 |
activeRequest.removeListener('close', onRequestClose); |
| 201 |
} |
| 202 |
}; |
| 203 |
|
| 204 |
const onRequestClose = () => { |
| 205 |
cleanup(); |
| 206 |
retryRequest({}); |
| 207 |
}; |
| 208 |
|
| 209 |
const cleanup = () => { |
| 210 |
activeRequest.removeListener('close', onRequestClose); |
| 211 |
activeResponse?.removeListener('data', onData); |
| 212 |
activeDecodedStream?.removeListener('end', onEnd); |
| 213 |
}; |
| 214 |
|
| 215 |
const onData = (chunk: Buffer) => { downloaded += chunk.length; }; |
| 216 |
|
| 217 |
const onEnd = () => { |
| 218 |
cleanup(); |
| 219 |
if (!reconnectIfEndedEarly()) { |
| 220 |
stream.end(); |
| 221 |
} |
| 222 |
}; |
| 223 |
|
| 224 |
activeRequest = httpLib.request(parsed, (res: IncomingMessage) => { |
| 225 |
// Needed for node v10, v12. |
| 226 |
// istanbul ignore next |
| 227 |
if (stream.destroyed) { return; } |
| 228 |
if (redirectStatusCodes.has(res.statusCode as number)) { |
| 229 |
if (redirects++ >= opts.maxRedirects) { |
| 230 |
stream.emit('error', new Miniget.MinigetError('Too many redirects')); |
| 231 |
} else { |
| 232 |
if (res.headers.location) { |
| 233 |
url = res.headers.location; |
| 234 |
} else { |
| 235 |
let err = new Miniget.MinigetError('Redirect status code given with no location', res.statusCode); |
| 236 |
stream.emit('error', err); |
| 237 |
cleanup(); |
| 238 |
return; |
| 239 |
} |
| 240 |
setTimeout(doDownload, parseInt(res.headers['retry-after'] || '0', 10) * 1000); |
| 241 |
stream.emit('redirect', url); |
| 242 |
} |
| 243 |
cleanup(); |
| 244 |
return; |
| 245 |
|
| 246 |
// Check for rate limiting. |
| 247 |
} else if (retryStatusCodes.has(res.statusCode as number)) { |
| 248 |
if (!retryRequest({ retryAfter: parseInt(res.headers['retry-after'] || '0', 10) })) { |
| 249 |
let err = new Miniget.MinigetError(`Status code: ${res.statusCode}`, res.statusCode); |
| 250 |
stream.emit('error', err); |
| 251 |
} |
| 252 |
cleanup(); |
| 253 |
return; |
| 254 |
} else if (res.statusCode && (res.statusCode < 200 || res.statusCode >= 400)) { |
| 255 |
let err = new Miniget.MinigetError(`Status code: ${res.statusCode}`, res.statusCode); |
| 256 |
if (res.statusCode >= 500) { |
| 257 |
onError(err); |
| 258 |
} else { |
| 259 |
stream.emit('error', err); |
| 260 |
} |
| 261 |
cleanup(); |
| 262 |
return; |
| 263 |
} |
| 264 |
|
| 265 |
activeDecodedStream = res as unknown as Transform; |
| 266 |
if (opts.acceptEncoding && res.headers['content-encoding']) { |
| 267 |
for (let enc of res.headers['content-encoding'].split(', ').reverse()) { |
| 268 |
let fn = opts.acceptEncoding[enc]; |
| 269 |
if (fn) { |
| 270 |
activeDecodedStream = activeDecodedStream.pipe(fn()); |
| 271 |
activeDecodedStream.on('error', onError); |
| 272 |
} |
| 273 |
} |
| 274 |
} |
| 275 |
if (!contentLength) { |
| 276 |
contentLength = parseInt(`${res.headers['content-length']}`, 10); |
| 277 |
acceptRanges = res.headers['accept-ranges'] === 'bytes' && |
| 278 |
contentLength > 0 && opts.maxReconnects > 0; |
| 279 |
} |
| 280 |
res.on('data', onData); |
| 281 |
activeDecodedStream.on('end', onEnd); |
| 282 |
activeDecodedStream.pipe(stream, { end: !acceptRanges }); |
| 283 |
activeResponse = res; |
| 284 |
stream.emit('response', res); |
| 285 |
res.on('error', onError); |
| 286 |
forwardEvents(res, responseEvents); |
| 287 |
}); |
| 288 |
activeRequest.on('error', onError); |
| 289 |
activeRequest.on('close', onRequestClose); |
| 290 |
forwardEvents(activeRequest, requestEvents); |
| 291 |
if (stream.destroyed) { |
| 292 |
streamDestroy(...destroyArgs); |
| 293 |
} |
| 294 |
stream.emit('request', activeRequest); |
| 295 |
activeRequest.end(); |
| 296 |
}; |
| 297 |
|
| 298 |
stream.abort = (err?: Error) => { |
| 299 |
console.warn('`MinigetStream#abort()` has been deprecated in favor of `MinigetStream#destroy()`'); |
| 300 |
stream.aborted = true; |
| 301 |
stream.emit('abort'); |
| 302 |
stream.destroy(err); |
| 303 |
}; |
| 304 |
|
| 305 |
let destroyArgs: any[] = []; |
| 306 |
const streamDestroy = (err?: Error) => { |
| 307 |
activeRequest.destroy(err); |
| 308 |
activeDecodedStream?.unpipe(stream); |
| 309 |
activeDecodedStream?.destroy(); |
| 310 |
clearTimeout(retryTimeout); |
| 311 |
}; |
| 312 |
|
| 313 |
stream._destroy = (...args: any[]) => { |
| 314 |
stream.destroyed = true; |
| 315 |
if (activeRequest) { |
| 316 |
streamDestroy(...args); |
| 317 |
} else { |
| 318 |
destroyArgs = args; |
| 319 |
} |
| 320 |
}; |
| 321 |
|
| 322 |
stream.text = () => new Promise((resolve, reject) => { |
| 323 |
let body = ''; |
| 324 |
stream.setEncoding('utf8'); |
| 325 |
stream.on('data', chunk => body += chunk); |
| 326 |
stream.on('end', () => resolve(body)); |
| 327 |
stream.on('error', reject); |
| 328 |
}); |
| 329 |
|
| 330 |
process.nextTick(doDownload); |
| 331 |
return stream; |
| 332 |
} |
| 333 |
|
| 334 |
export = Miniget; |