fix: retry on ECONNRESET

src/helpers/util.ts

@@ -32,3 +32,5 @@ export function argAsArray<T>(args?: T | T[]): T[] {
   }
   return result.concat(args)
 }
+
+export function sleep(ms: number) { return new Promise(r => setTimeout(r, ms)); }

src/helpers/web.ts

@@ -1,9 +1,6 @@
 import dns from 'node:dns'
 import { snakeCase } from 'snake-case'
-import {
-  request,
-  setGlobalDispatcher,
-} from 'undici'
+import { request, setGlobalDispatcher, errors, Dispatcher } from 'undici'
 
 import { version } from '../../package.json'
 import {
@@ -15,8 +12,13 @@ import {
   UploaderEnvs,
   UploaderInputs,
 } from '../types'
-import { info } from './logger'
+import { UploadLogger, info, logError } from './logger'
 import { addProxyIfNeeded } from './proxy'
+import { sleep } from './util'
+
+
+const maxRetries = 4
+const baseBackoffDelayMs = 1000 // Adjust this value based on your needs.
 
 /**
  *
@@ -33,7 +35,7 @@ export function populateBuildParams(
   serviceParams.name = args.name || envs.CODECOV_NAME || ''
   serviceParams.tag = args.tag || ''
 
-  if (typeof args.flags === "string") {
+  if (typeof args.flags === 'string') {
     serviceParams.flags = args.flags
   } else {
     serviceParams.flags = args.flags.join(',')
@@ -51,12 +53,36 @@ export function getPackage(source: string): string {
   }
 }
 
+async function requestWithRetry(
+  url: string,
+  options: Dispatcher.RequestOptions,
+  retryCount = 0,
+): Promise<Dispatcher.ResponseData> {
+  try {
+    const response = await request(url, options)
+    return response
+  } catch (error: unknown) {
+    if (
+      ((error instanceof errors.UndiciError && error.code == 'ECONNRESET') ||
+        error instanceof errors.ConnectTimeoutError ||
+        error instanceof errors.SocketError) &&
+      retryCount < maxRetries
+    ) {
+      const backoffDelay = baseBackoffDelayMs * 2 ** retryCount
+      await sleep(backoffDelay)
+      UploadLogger.verbose('Request to Codecov failed. Retrying...')
+      logError(`Request error: ${error.message}`)
+      return requestWithRetry(url, options, retryCount + 1)
+    }
+    throw error
+  }
+}
 
 export async function uploadToCodecovPUT(
   putAndResultUrlPair: PostResults,
   uploadFile: string | Buffer,
   envs: UploaderEnvs,
-  args: UploaderArgs
+  args: UploaderArgs,
 ): Promise<PutResults> {
   info('Uploading...')
 
@@ -69,8 +95,11 @@ export async function uploadToCodecovPUT(
   if (requestHeaders.agent) {
     setGlobalDispatcher(requestHeaders.agent)
   }
-  dns.setDefaultResultOrder('ipv4first');
-  const response = await request(requestHeaders.url.origin, requestHeaders.options)
+  dns.setDefaultResultOrder('ipv4first')
+  const response = await requestWithRetry(
+    requestHeaders.url.origin,
+    requestHeaders.options,
+  )
 
   if (response.statusCode !== 200) {
     const data = await response.body.text()
@@ -101,8 +130,12 @@ export async function uploadToCodecovPOST(
   if (requestHeaders.agent) {
     setGlobalDispatcher(requestHeaders.agent)
   }
-  dns.setDefaultResultOrder('ipv4first');
-  const response = await request(requestHeaders.url.origin, requestHeaders.options)
+  dns.setDefaultResultOrder('ipv4first')
+
+  const response = await requestWithRetry(
+    requestHeaders.url.origin,
+    requestHeaders.options,
+  )
 
   if (response.statusCode !== 200) {
     const data = await response.body.text()
@@ -113,7 +146,6 @@ export async function uploadToCodecovPOST(
 
   return await response.body.text()
 }
-
 /**
  *
  * @param {Object} queryParams
@@ -125,8 +157,6 @@ export function generateQuery(queryParams: Partial<IServiceParams>): string {
   ).toString()
 }
 
-
-
 export function parsePOSTResults(putAndResultUrlPair: string): PostResults {
   info(putAndResultUrlPair)
 
@@ -136,7 +166,9 @@ export function parsePOSTResults(putAndResultUrlPair: string): PostResults {
   const matches = putAndResultUrlPair.match(re)
 
   if (matches === null) {
-    throw new Error(`Parsing results from POST failed: (${putAndResultUrlPair})`)
+    throw new Error(
+      `Parsing results from POST failed: (${putAndResultUrlPair})`,
+    )
   }
 
   if (matches?.length !== 2) {
@@ -147,12 +179,12 @@ export function parsePOSTResults(putAndResultUrlPair: string): PostResults {
 
   if (matches[0] === undefined || matches[1] === undefined) {
     throw new Error(
-      `Invalid URLs received when parsing results from POST: ${matches[0]},${matches[1]}`
+      `Invalid URLs received when parsing results from POST: ${matches[0]},${matches[1]}`,
     )
   }
   const resultURL = new URL(matches[0].trimEnd())
   const putURL = new URL(matches[1])
-   // This match may have trailing 0x0A and 0x0D that must be trimmed
+  // This match may have trailing 0x0A and 0x0D that must be trimmed
 
   return { putURL, resultURL }
 }
@@ -170,9 +202,10 @@ export function generateRequestHeadersPOST(
   envs: UploaderEnvs,
   args: UploaderArgs,
 ): IRequestHeaders {
-  const url = new URL(`upload/v4?package=${getPackage(
-    source,
-  )}&token=${token}&${query}`, postURL)
+  const url = new URL(
+    `upload/v4?package=${getPackage(source)}&token=${token}&${query}`,
+    postURL,
+  )
 
   const headers = {
     'X-Upload-Token': token,
@@ -191,8 +224,6 @@ export function generateRequestHeadersPOST(
   }
 }
 
-
-
 export function generateRequestHeadersPUT(
   uploadURL: URL,
   uploadFile: string | Buffer,

test/helpers/web.test.ts

@@ -3,6 +3,7 @@ import {
   MockClient,
   ProxyAgent,
   setGlobalDispatcher,
+  errors
 } from 'undici'
 
 import { version } from '../../package.json'
@@ -20,6 +21,9 @@ import {
 import { IServiceParams, PostResults, UploaderArgs, UploaderEnvs } from '../../src/types.js'
 import { createEmptyArgs } from '../test_helpers'
 
+import * as utilModule from '../../src/helpers/util'
+
+
 describe('Web Helpers', () => {
   let uploadURL: string
   let token: string
@@ -280,10 +284,6 @@ describe('Web Helpers', () => {
 
 
 
-
-
-
-
       https://put.codecov.local`
       const expectedResults = {
         putURL: 'https://put.codecov.local/',
@@ -292,6 +292,117 @@ describe('Web Helpers', () => {
       expect(JSON.stringify(parsePOSTResults(testURL))).toStrictEqual(JSON.stringify(expectedResults))
     })
   })
+
+  describe('Uploader should retry POST on ECONNRESET', () => {
+    it('should retry and return response data when ECONNRESET occurs once', async () => {
+      const envs: UploaderEnvs = {}
+      const args: UploaderArgs = {
+        flags: '',
+        slug: '',
+        upstream: '',
+      }
+      const error = new errors.UndiciError("conn reset error")
+      error.code = 'ECONNRESET'
+      uploadURL = 'http://codecov.io'
+      mockAgent.disableNetConnect()
+      mockClient = mockAgent.get(uploadURL)
+
+      mockClient.intercept({
+        method: 'POST',
+        path: `/upload/v4?package=uploader-${version}&token=${token}&hello`,
+      }).replyWithError(new errors.ConnectTimeoutError('timeout error')).times(1)
+
+      mockClient.intercept({
+        method: 'POST',
+        path: `/upload/v4?package=uploader-${version}&token=${token}&hello`,
+      }).replyWithError(error).times(1)
+
+      mockClient.intercept({
+        method: 'POST',
+        path: `/upload/v4?package=uploader-${version}&token=${token}&hello`,
+      }).reply(200, 'testPOSTHTTP')
+
+      const responseData = await uploadToCodecovPOST(new URL(uploadURL), token, query, source, envs, args);
+      try {
+        expect(responseData).toBe('testPOSTHTTP')
+      } catch (error) {
+        throw new Error(`${responseData} - ${error}`)
+      }
+    });
+
+    it('should fail with error if ECONNRESET happens 5 times', async () => {
+      const mockSleep = jest.spyOn(utilModule, 'sleep').mockResolvedValue(42)
+      const envs: UploaderEnvs = {}
+      const args: UploaderArgs = {
+        flags: '',
+        slug: '',
+        upstream: '',
+      }
+      const error = new errors.UndiciError("conn reset error")
+      error.code = 'ECONNRESET'
+      uploadURL = 'http://codecov.io'
+      mockAgent.disableNetConnect()
+      mockClient = mockAgent.get(uploadURL)
+
+      mockClient.intercept({
+        method: 'POST',
+        path: `/upload/v4?package=uploader-${version}&token=${token}&hello`,
+      }).replyWithError(error).times(5)
+
+      mockClient.intercept({
+        method: 'POST',
+        path: `/upload/v4?package=uploader-${version}&token=${token}&hello`,
+      }).reply(200, 'testPOSTHTTP')
+
+      try {
+        await uploadToCodecovPOST(new URL(uploadURL), token, query, source, envs, args);
+        expect(true).toBe(false)
+      } catch (error) {
+        expect(error).toBeInstanceOf(errors.UndiciError)
+        expect(mockSleep).toBeCalledTimes(4)
+      }
+    })
+  });
+
+  describe('Uploader should retry PUT on ECONNRESET', () => {
+    it('should retry and return response data when ECONNRESET occurs once', async () => {
+      // Replace the original setTimeout with fakeSetTimeout
+      const envs: UploaderEnvs = {}
+    const args: UploaderArgs = {
+      flags: '',
+      slug: '',
+      upstream: '',
+    }
+    jest.spyOn(console, 'log').mockImplementation(() => void {})
+    const postResults: PostResults = {
+      putURL: new URL('https://codecov.io'),
+      resultURL: new URL('https://results.codecov.io'),
+    }
+
+      const error = new errors.UndiciError("conn reset error")
+      error.code = 'ECONNRESET'
+      mockAgent.disableNetConnect()
+      mockClient = mockAgent.get(postResults.putURL.origin)
+
+      mockClient.intercept({
+        method: 'PUT',
+        path: `/`,
+      }).replyWithError(new errors.ConnectTimeoutError('timeout error')).times(1)
+
+      mockClient.intercept({
+        method: 'PUT',
+        path: `/`,
+      }).replyWithError(error).times(1)
+
+      mockClient.intercept({
+        method: 'PUT',
+        path: `/`,
+      }).reply(200, postResults.resultURL.href)
+
+      const response = await uploadToCodecovPUT(postResults, uploadFile, envs, args)
+      expect(response.resultURL.href).toStrictEqual('https://results.codecov.io/')
+    });
+  });
 })
 
 describe('displayChangelog()', () => {