port node-fetch tests + fixes

Author: ronag

README.md

@@ -157,7 +157,7 @@ See [Dispatcher.connect](docs/api/Dispatcher.md#dispatcherconnect) for more deta
 
 https://fetch.spec.whatwg.org/
 
-### `undici.fetch([url, options]): Promise`
+### `undici.fetch(resource[, init]): Promise`
 
 Implements [fetch](https://fetch.spec.whatwg.org/).
 
@@ -167,8 +167,8 @@ This is [experimental](https://nodejs.org/api/documentation.html#documentation_s
 
 Arguments:
 
-* **url** `string | URL | object`
-* **options** `RequestInit`
+* **resource** `string | URL`
+* **init** `RequestInit`
 
 Returns: `Promise<Response>`
 

index.js

@@ -84,7 +84,25 @@ function makeDispatcher (fn) {
 module.exports.setGlobalDispatcher = setGlobalDispatcher
 module.exports.getGlobalDispatcher = getGlobalDispatcher
 
-module.exports.fetch = makeDispatcher(api.fetch)
+if (api.fetch) {
+  const _fetch = makeDispatcher(api.fetch)
+  module.exports.fetch = async function fetch (resource, init) {
+    try {
+      return await _fetch(resource, init)
+    } catch (err) {
+      // TODO (fix): This is a little weird. Spec compliant?
+      if (err.code === 'ERR_INVALID_URL') {
+        const er = new TypeError('Invalid URL')
+        er.cause = err
+        throw er
+      }
+      throw err
+    }
+  }
+  module.exports.Headers = api.fetch.Headers
+  module.exports.Response = api.fetch.Response
+}
+
 module.exports.request = makeDispatcher(api.request)
 module.exports.stream = makeDispatcher(api.stream)
 module.exports.pipeline = makeDispatcher(api.pipeline)

lib/api/api-fetch/body.js

@@ -1,9 +1,11 @@
 'use strict'
 
 const util = require('../../core/util')
-const { Readable } = require('stream')
+const { finished } = require('stream')
+const { AbortError } = require('../../core/errors')
 
-let TransformStream
+let ReadableStream
+let CountQueuingStrategy
 
 // https://fetch.spec.whatwg.org/#concept-bodyinit-extract
 function extractBody (body) {
@@ -24,6 +26,10 @@ function extractBody (body) {
       source: body
     }, 'text/plain;charset=UTF-8']
   } else if (body instanceof ArrayBuffer || ArrayBuffer.isView(body)) {
+    if (body instanceof DataView) {
+      // TODO: Blob doesn't seem to work with DataView?
+      body = body.buffer
+    }
     return [{
       source: body
     }, null]
@@ -39,20 +45,12 @@ function extractBody (body) {
 
     let stream
     if (util.isStream(body)) {
-      stream = Readable.toWeb(body)
+      stream = toWeb(body)
     } else {
       if (body.locked) {
         throw new TypeError('locked')
       }
-
-      if (!TransformStream) {
-        TransformStream = require('stream/web').TransformStream
-      }
-
-      // https://streams.spec.whatwg.org/#readablestream-create-a-proxy
-      const identityTransform = new TransformStream()
-      body.pipeThrough(identityTransform)
-      stream = identityTransform
+      stream = body
     }
 
     return [{
@@ -63,4 +61,74 @@ function extractBody (body) {
   }
 }
 
+function toWeb (streamReadable) {
+  if (!ReadableStream) {
+    ReadableStream = require('stream/web').ReadableStream
+  }
+  if (!CountQueuingStrategy) {
+    CountQueuingStrategy = require('stream/web').CountQueuingStrategy
+  }
+
+  if (util.isDestroyed(streamReadable)) {
+    const readable = new ReadableStream()
+    readable.cancel()
+    return readable
+  }
+
+  const objectMode = streamReadable.readableObjectMode
+  const highWaterMark = streamReadable.readableHighWaterMark
+  // When not running in objectMode explicitly, we just fall
+  // back to a minimal strategy that just specifies the highWaterMark
+  // and no size algorithm. Using a ByteLengthQueuingStrategy here
+  // is unnecessary.
+  const strategy = objectMode
+    ? new CountQueuingStrategy({ highWaterMark })
+    : { highWaterMark }
+
+  let controller
+
+  function onData (chunk) {
+    // Copy the Buffer to detach it from the pool.
+    if (Buffer.isBuffer(chunk) && !objectMode) {
+      chunk = new Uint8Array(chunk)
+    }
+    controller.enqueue(chunk)
+    if (controller.desiredSize <= 0) {
+      streamReadable.pause()
+    }
+  }
+
+  streamReadable.pause()
+
+  finished(streamReadable, (err) => {
+    if (err && err.code === 'ERR_STREAM_PREMATURE_CLOSE') {
+      const er = new AbortError()
+      er.cause = er
+      err = er
+    }
+
+    if (err) {
+      controller.error(err)
+    } else {
+      controller.close()
+    }
+  })
+
+  streamReadable.on('data', onData)
+
+  return new ReadableStream({
+    start (c) {
+      controller = c
+    },
+
+    pull () {
+      streamReadable.resume()
+    },
+
+    cancel (reason) {
+      util.destroy(streamReadable, reason)
+    }
+  }, strategy)
+}
+
 module.exports = { extractBody }

lib/api/api-fetch/headers.js

@@ -5,7 +5,11 @@
 const { types } = require('util')
 const { validateHeaderName, validateHeaderValue } = require('http')
 const { kHeadersList } = require('../../core/symbols')
-const { InvalidHTTPTokenError, HTTPInvalidHeaderValueError, InvalidArgumentError, InvalidThisError } = require('../../core/errors')
+const {
+  InvalidHTTPTokenError,
+  HTTPInvalidHeaderValueError,
+  InvalidThisError
+} = require('../../core/errors')
 
 function binarySearch (arr, val) {
   let low = 0
@@ -49,21 +53,21 @@ function isHeaders (object) {
 function fill (headers, object) {
   if (isHeaders(object)) {
     // Object is instance of Headers
-    headers[kHeadersList] = Array.splice(object[kHeadersList])
+    headers[kHeadersList] = [...object[kHeadersList]]
   } else if (Array.isArray(object)) {
     // Support both 1D and 2D arrays of header entries
     if (Array.isArray(object[0])) {
       // Array of arrays
       for (let i = 0; i < object.length; i++) {
         if (object[i].length !== 2) {
-          throw new InvalidArgumentError(`The argument 'init' is not of length 2. Received ${object[i]}`)
+          throw new TypeError(`The argument 'init' is not of length 2. Received ${object[i]}`)
         }
         headers.append(object[i][0], object[i][1])
       }
     } else if (typeof object[0] === 'string' || Buffer.isBuffer(object[0])) {
       // Flat array of strings or Buffers
       if (object.length % 2 !== 0) {
-        throw new InvalidArgumentError(`The argument 'init' is not even in length. Received ${object}`)
+        throw new TypeError(`The argument 'init' is not even in length. Received ${object}`)
       }
       for (let i = 0; i < object.length; i += 2) {
         headers.append(
@@ -73,7 +77,7 @@ function fill (headers, object) {
       }
     } else {
       // All other array based entries
-      throw new InvalidArgumentError(`The argument 'init' is not a valid array entry. Received ${object}`)
+      throw new TypeError(`The argument 'init' is not a valid array entry. Received ${object}`)
     }
   } else if (!types.isBoxedPrimitive(object)) {
     // Object of key/value entries
@@ -94,12 +98,20 @@ class Headers {
   constructor (init = {}) {
     // validateObject allowArray = true
     if (!Array.isArray(init) && typeof init !== 'object') {
-      throw new InvalidArgumentError('The argument \'init\' must be one of type Object or Array')
+      throw new TypeError('The argument \'init\' must be one of type Object or Array')
     }
     this[kHeadersList] = []
     fill(this, init)
   }
 
+  get [Symbol.toStringTag] () {
+    return this.constructor.name
+  }
+
+  toString () {
+    return Object.prototype.toString.call(this)
+  }
+
   append (...args) {
     if (!isHeaders(this)) {
       throw new InvalidThisError('Header')
@@ -233,8 +245,31 @@ class Headers {
       callback.call(thisArg, this[kHeadersList][index + 1], this[kHeadersList][index], this)
     }
   }
+
+  [Symbol.for('nodejs.util.inspect.custom')] () {
+    return Object.fromEntries(this.entries())
+  }
 }
 
+// Re-shaping object for Web IDL tests.
+Object.defineProperties(
+  Headers.prototype,
+  [
+    'append',
+    'delete',
+    'entries',
+    'forEach',
+    'get',
+    'has',
+    'keys',
+    'set',
+    'values'
+  ].reduce((result, property) => {
+    result[property] = { enumerable: true }
+    return result
+  }, {})
+)
+
 Headers.prototype[Symbol.iterator] = Headers.prototype.entries
 
 module.exports = Headers

lib/api/api-fetch/index.js

@@ -4,16 +4,19 @@
 
 const Headers = require('./headers')
 const { kHeadersList } = require('../../core/symbols')
-const { METHODS } = require('http')
+const { METHODS, STATUS_CODES } = require('http')
 const Response = require('./response')
+
 const {
   InvalidArgumentError,
   NotSupportedError,
   RequestAbortedError
 } = require('../../core/errors')
 const { addSignal, removeSignal } = require('../abort-signal')
 const { extractBody } = require('./body')
+const { kUrlList } = require('./symbols')
 
+let TransformStream
 let ReadableStream
 
 class FetchHandler {
@@ -69,46 +72,42 @@ class FetchHandler {
     let response
     if (headers.has('location')) {
       if (this.redirect === 'manual') {
-        response = new Response({
+        response = new Response(null, {
           type: 'opaqueredirect',
+          status: 0,
           url: this.url
         })
       } else {
-        response = new Response({
-          type: 'error',
-          url: this.url
-        })
+        response = Response.error()
       }
     } else {
       const self = this
       if (!ReadableStream) {
         ReadableStream = require('stream/web').ReadableStream
       }
-      response = new Response({
-        type: 'default',
-        url: this.url,
-        body: new ReadableStream({
-          async start (controller) {
-            self.controller = controller
-          },
-          async pull () {
-            resume()
-          },
-          async cancel (reason) {
-            let err
-            if (reason instanceof Error) {
-              err = reason
-            } else if (typeof reason === 'string') {
-              err = new Error(reason)
-            } else {
-              err = new RequestAbortedError()
-            }
-            abort(err)
+      response = new Response(new ReadableStream({
+        async start (controller) {
+          self.controller = controller
+        },
+        async pull () {
+          resume()
+        },
+        async cancel (reason) {
+          let err
+          if (reason instanceof Error) {
+            err = reason
+          } else if (typeof reason === 'string') {
+            err = new Error(reason)
+          } else {
+            err = new RequestAbortedError()
           }
-        }, { highWaterMark: 16384 }),
-        statusCode,
+          abort(err)
+        }
+      }, { highWaterMark: 16384 }), {
+        status: statusCode,
+        statusText: STATUS_CODES[statusCode],
         headers,
-        context
+        [kUrlList]: [this.url, ...((context && context.history) || [])]
       })
     }
 
@@ -182,7 +181,11 @@ async function fetch (opts) {
   }
 
   if (opts.redirect != null) {
-    // TODO: Validate
+    if (
+      typeof opts.redirect !== 'string' ||
+      !/^(follow|manual|error)/.test(opts.redirect)) {
+      throw new TypeError(`Redirect option '${opts.redirect}' is not a valid value of RequestRedirect`)
+    }
   } else {
     opts.redirect = 'follow'
   }
@@ -214,6 +217,17 @@ async function fetch (opts) {
 
   const [body, contentType] = extractBody(opts.body)
 
+  if (body.stream) {
+    if (!TransformStream) {
+      TransformStream = require('stream/web').TransformStream
+    }
+
+    // https://streams.spec.whatwg.org/#readablestream-create-a-proxy
+    const identityTransform = new TransformStream()
+    body.pipeThrough(identityTransform)
+    body.stream = identityTransform
+  }
+
   if (contentType && !headers.has('content-type')) {
     headers.set('content-type', contentType)
   }