Stream File URL to AWS Pre-signed URL

You can write a UploadService file.

The below code (Nodejs) uploads a file from a given URL to an AWS S3 bucket using a pre-signed URL. It streams the file instead of loading it fully into memory.

The uploadFileToPresignedUrl(presignedUrl, fileUrl) function does the actual upload.

• It first makes a GET request to the fileUrl using axios, with the response type set to 'stream'.
• The response from this request, which is a stream of the file data, is then used as the data in a PUT request to the presignedUrl. The headers of this request include the content type and length of the file.
• The PUT request also uses a httpsAgent with keepAlive set to true.
• The maxContentLength is set to 5GB and maxBodyLength is set to Infinity.

Here is the full source code:

"use strict";
const axios = require("axios");
const https = require("https");

async function uploadFileToPresignedUrl(presignedUrl, fileUrl) {
  const response = await axios({
    method: "get",
    url: fileUrl,
    responseType: "stream",
    httpsAgent: new https.Agent({ keepAlive: true }),
  });

  const fileStream = response.data;
  const uploadResponse = await axios({
    method: "put",
    url: presignedUrl,
    data: fileStream,
    headers: {
      "Content-Type": response.headers["content-type"],
      "Content-Length": response.headers["content-length"],
    },
    httpsAgent: new https.Agent({ keepAlive: true }),
    maxContentLength: 5 * 1024 * 1024 * 1024, // 1GB
    maxBodyLength: Infinity,
  });

  return uploadResponse;
}

class UploadService {
  constructor() {
    this.uploadStream = this.uploadStream.bind(this);
  }

  async uploadStream(presignedUrl, fileUrl) {
    try {
      const response = await uploadFileToPresignedUrl(presignedUrl, fileUrl);
      return true;
    } catch (error) {
      throw error;
    }
  }
}

module.exports = UploadService;