Download file from url and upload it to AWS S3 without saving - node.js

I'm writing an application which downloads images from a url and then uploads it to an S3 bucket using the [aws-sdk](

[To see links please register here]

).

Perviously I was just downloading images and saving them to disk like this.

request.head(url, function(err, res, body){

request(url).pipe(fs.createWriteStream(image_path));

});

And then uploading the images to AWS S3 like this

fs.readFile(image_path, function(err, data){
s3.client.putObject({
Bucket: 'myBucket',
Key: image_path,
Body: data
ACL:'public-read'
}, function(err, resp) {
if(err){
console.log("error in s3 put object cb");
} else {
console.log(resp);
console.log("successfully added image to s3");
}
});
});

But I would like to skip the part where I save the image to disk. Is there some way I can `pipe` the response from `request(url)` to a variable and then upload that?

Here's some javascript that does this nicely:

var options = {
uri: uri,
encoding: null
};
request(options, function(error, response, body) {
if (error || response.statusCode !== 200) {
console.log("failed to get image");
console.log(error);
} else {
s3.putObject({
Body: body,
Key: path,
Bucket: 'bucket_name'
}, function(error, data) {
if (error) {
console.log("error downloading image to s3");
} else {
console.log("success uploading to s3");
}
});
}
});

This is what I did and works nicely:

<!-- begin snippet: js hide: false console: true babel: false -->

<!-- language: lang-js -->

const request = require('request-promise')
const AWS = require('aws-sdk')
const s3 = new AWS.S3()

const options = {
uri: uri,
encoding: null
};

async load() {

const body = await request(options)

const uploadResult = await s3.upload({
Bucket: 'bucket_name',
Key : path,
Body : body,
}).promise()

}



<!-- end snippet -->

What about something like that:

```javascript
const stream = require('stream');
const request = require('request');
const s3 = new AWS.S3()

const pass = new stream.PassThrough();

request(url).pipe(pass);

s3.upload({
Bucket: 'bucket_name',
Key: path,
Body: pass,
});

```

You can implement with Axios like this. Refer [this][1] for more info.


const axios = require("axios");
const AWS = require("aws-sdk");
const { PassThrough } = require("stream");

const s3 = new AWS.S3({
accessKeyId: "accessKeyId",
secretAccessKey: "accessKey",
region: "region",
});

const bucket = "BucketName";
const key = "key";

const uploadToS3 = async (bucket, key) => {
try {
const stream = await axios.get(url, { responseType: "stream" });

const passThrough = new PassThrough();

const response = s3.upload({ Bucket: bucket, Key: key, Body: passThrough });

stream.data.pipe(passThrough);

return response.then((data) => data.Location).catch((e) => console.error(e));
} catch (error) {
console.error(error);
}
};

uploadToS3(bucket, key);


[1]:

[To see links please register here]

import axios from "axios";
import aws from 'aws-sdk'
import crypto from 'crypto'

const s3 = new aws.S3();

export const urlToS3 = async ({ url, bucket = "rememoio-users", key = Date.now() + crypto.randomBytes(8).toString('hex') + ".png" }) => {
try {
const { data } = await axios.get(url, { responseType: "stream" });

const upload = await s3.upload({
Bucket: bucket,
ACL: 'public-read',
Key: key,
Body: data,
}).promise();

return upload.Location;
} catch (error) {
console.error(error);
throw new Error;
}
};

using fetch:

//fetch image from url
const imageResp = await fetch(
'<image url>'
)
// transform to arrayBuffer
const imageBinaryBuffer = Buffer.from(await imageResp.arrayBuffer())
//get image type
const imageType = imageName.toLowerCase().includes(".png")
? "image/png"
: "image/jpg";

//get presigned url and data [this can be different on your end]
const presignedResponse = await getPresignedURL(imageBinaryBuffer, imageName, imageType)

const s3Result = presignedResponse.data
// build the formData
let formData = new FormData()
Object.keys(s3Result.fields).forEach(key => {
formData.append(key, s3Result.fields[key]);
});
formData.append("file", imageBinaryBuffer);

const s3resp = await fetch(s3Result.url, {
method: "POST",
body: formData,
});

return s3resp.headers.location