The example below shows how to store and serve uploaded file via S3. This also covers removing the files from S3 when removed from FilesCollection.
First, install aws-sdk Official AWS SDK Docs
npm install --save aws-sdkOr:
meteor npm install --save aws-sdkTypical regions are these, see full list at AWS S3 console. Every region is supported.
- US Standard (default):
us-standard - US West (Oregon):
us-west-2 - US West (Northern California):
us-west-1 - EU (Ireland):
eu-west-1 - EU (Frankfurt):
eu-central-1 - Asia Pacific (Singapore):
ap-southeast-1 - Asia Pacific (Tokyo):
ap-northeast-1 - South America (Sao Paulo):
sa-east-1
Prepare: Get access to AWS S3:
- Go to aws.amazon.com/s3 (Sign(in|up) if required)
- Follow the steps from this procedure - (to the step where policy and user is created)
- Create an S3 bucket in preferred region
- Get an "Access Key Id" and "Secret Key"
- Note that in order to delete files from S3 you will additionally have to add the action "s3:DeleteObject" to the created policy.
First, create the settings.json file and add AWS:S3 credentials to it:
Use it with: meteor --settings settings.json
{
"s3": {
"key": "AWSKEY",
"secret": "AWSSECRET",
"bucket": "BUCKETNAME",
"region": "us-west-1"
}
}Instead of using settings.json, - environment variable can be used:
import { Meteor } from 'meteor/meteor';
/** env.var example: S3='{"s3":{"key": "xxx", "secret": "xxx", "bucket": "xxx", "region": "xxx""}}' **/
if (process.env.S3) {
Meteor.settings.app.s3 = JSON.parse(process.env.S3).s3;
}File: Server-side-file-store.js.
Use this at Meteor's imports/server directory, NOT on the client
import { Meteor } from 'meteor/meteor';
import { _ } from 'meteor/underscore';
import { Random } from 'meteor/random';
import { FilesCollection } from 'meteor/ostrio:files';
import stream from 'node:stream';
import S3 from 'aws-sdk/clients/s3'; /* http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html */
/* See fs-extra and graceful-fs NPM packages */
/* For better i/o performance */
import fs from 'node:fs';
/* Example: S3='{"s3":{"key": "xxx", "secret": "xxx", "bucket": "xxx", "region": "xxx""}}' meteor */
if (process.env.S3) {
Meteor.settings.app.s3 = JSON.parse(process.env.S3).s3;
}
const s3Conf = Meteor.settings.app.s3 || {};
/* Check settings existence in `Meteor.settings` */
/* This is the best practice for app security */
if (!s3Conf || !s3Conf?.key || !s3Conf?.secret || !s3Conf?.bucket || !s3Conf?.region) {
throw new Meteor.Error(401, 'Missing Meteor file settings');
}
// Create a new S3 object
const s3 = new S3({
secretAccessKey: s3Conf.secret,
accessKeyId: s3Conf.key,
region: s3Conf.region,
// sslEnabled: true, // optional
httpOptions: {
timeout: 6000,
agent: false
}
});
// Declare the Meteor file collection on the Server
const UserFiles = new FilesCollection({
debug: false, // Change to `true` for debugging
storagePath: 'assets/app/uploads/uploadedFiles',
collectionName: 'userFiles',
// Disallow Client to execute remove, use the Meteor.method
allowClientCode: false,
// Start moving files to AWS:S3
// after fully received by the Meteor server
onAfterUpload(fileRef) {
// Run through each of the uploaded file
for(let version in fileRef.versions) {
if (!fileRef.versions[version]) {
continue;
}
// Clone version's object
const vRef = _.clone(fileRef.versions[version]);
// We use Random.id() instead of real file's _id
// to secure files from reverse engineering
// As after viewing this code it will be easy
// to get access to unlisted and protected files
const filePath = `files/${Random.id()}-${version}.${fileRef.extension}`;
const stream = fs.createReadStream(vRef.path);
stream.on('error', (error) => {
console.error('[afterUpload] [createReadStream] [ERROR:] File was not uploaded to S3', fileRef._id, error);
});
client.putObject({
StorageClass: 'STANDARD',
Bucket: s3Conf.bucket,
Key: filePath,
Body: stream,
ContentType: vRef.type,
}, async (error) => {
if (error) {
console.error('[afterUpload] [putObject] Error:', fileRef._id, error);
return;
}
try {
await this.collection.updateAsync({ _id: fileRef._id }, {
$set: {
[`versions.${version}.meta.pipePath`]: filePath
}
});
// Unlink original file from FS
// after successful upload to AWS:S3
await this.unlinkAsync(fileRef, version);
} catch (_unlinkError) {
// If file was removed before it was fully moved to S3
// `.updateAsync()` or `.unlinkAsync()` with throw an error
// Then we will need to remove that file from S3
client.deleteObject({
Bucket: s3Conf.bucket,
Key: filePath,
}, (deleteError) => {
if (deleteError) {
console.error('[afterUpload] [deleteObject] Error:', fileRef._id, deleteError);
} else {
console.info('[afterUpload] [deleteObject] unlinked file successfully removed from S3', fileRef._id);
}
});
}
});
}
},
// Intercept calls to `.remove()` and `.removeAsync()` to remove file from S3
// onAfterRemove called right after record is removed from the MongoDB Collection
// and before calling `.unlink()` or `.unlinkAsync()`,
// return `true` to prevent calling `.unlink()` or `.unlinkAsync()`
onAfterRemove(docs) {
for (let i = docs.length - 1; i >= 0; i--) {
if (docs[i].versions) {
for (let version in docs[i].versions) {
if (docs[i].versions[version]) {
const vRef = docs[i].versions[version];
if (vRef?.meta?.pipePath) {
client.deleteObject({
Bucket: s3Conf.bucket,
Key: vRef.meta.pipePath,
}, (error) => {
if (error) {
console.error('[onAfterRemove] [deleteObject] Error:', error);
} else {
console.info('[onAfterRemove] [deleteObject] Successfully removed from S3', vRef.path, vRef.meta.pipePath);
}
});
}
}
}
}
}
if (docs.length === 1 && docs[0].versions.original?.meta?.pipePath) {
// RETURN true TO AVOID UNNECESSARY .unlinkAsync AS FILE WAS ALREADY REMOVED FROM FS AFTER IT WAS UPLOADED TO S3
return true;
}
return false;
},
// Intercept access to the file
// And redirect request to AWS:S3
interceptDownload(http, fileRef, version) {
let path;
if (fileRef && fileRef.versions && fileRef.versions[version] && fileRef.versions[version].meta && fileRef.versions[version].meta.pipePath) {
path = fileRef.versions[version].meta.pipePath;
}
if (!path) {
// While file is not yet uploaded to AWS:S3
// It will be served file from FS
return false;
}
// If file is successfully moved to AWS:S3
// We will pipe request to AWS:S3
// So, original link will stay always secure
// To force ?play and ?download parameters
// and to keep original file name, content-type,
// content-disposition, chunked "streaming" and cache-control
// we're using low-level .serve() method
const opts = {
Bucket: s3Conf.bucket,
Key: path
};
if (http.request.headers.range) {
const vRef = fileRef.versions[version];
let range = _.clone(http.request.headers.range);
const array = range.split(/bytes=([0-9]*)-([0-9]*)/);
const start = parseInt(array[1]);
let end = parseInt(array[2]);
if (isNaN(end)) {
// Request data from AWS:S3 by small chunks
end = (start + this.chunkSize) - 1;
if (end >= vRef.size) {
end = vRef.size - 1;
}
}
opts.Range = `bytes=${start}-${end}`;
http.request.headers.range = `bytes=${start}-${end}`;
}
const responseEnd = (error) => {
if (error) {
console.error('[interceptDownload] [responseEnd]', error);
}
if (!http.response.finished) {
http.response.end();
}
};
const awsStream = client.getObject(opts).createReadStream();
awsStream.on('error', responseEnd);
this.serve(http, fileRef, fileRef.versions[version], version, awsStream);
return true;
}
});The basic concept: you already have a S3 folder that you use for storage above. We are going to set a Lambda trigger on that folder and for each file saved into (plus any other condition you wish), we will save a thumb into another folder.
First, sign in to your AWS console and select your region from the top bar. Go to your Lambda dashboard and create a new function.
Add a trigger for S3, select your bucket, select "Object Created(All)", check Enable trigger and save (Add). Then add the "Function Code". The code will be your xxx.js file zipped together with the node_modules folder used by your xxx.js file. Please note that your Lambda function will need to have the same name as your xxx.js file (e.g. JS file name: ImageResizer.js will require the Lambda function name/handler ImageResizer.handler. Upload your ZIP file.
We will be using two differents methods so please feel free to chose the one you prefer.
- Official Lambda resizer by AWS: full documentation here. This is based on sharp.js, claimed to be 4-5 times faster than ImageMagick. Just download the ZIP from the Amazon documentation and follow the steps above. You might want to make sure that the packages in the package.json file are at the toppes version. If not, please run an npm install to latest versions in order to generate the updated node_modules before you zip your index.js and node_modules folder together.
- Resizer based on ImageMagic (example shows a resize to output JPG, 420px width, 85% quality, with a meta attached for CachControl set to 10 days).
{
"name": "amazon-lambda-resizer",
"version": "0.0.1",
"description": "Resizer for lambda images in a S3 bucket from a source_folder to target_folder",
"main": "index.js",
"scripts": {
"start": "node index.js"
},
"dependencies": {
"async": "^2.6.0",
"aws-sdk": "^2.240.1",
"gm": "^1.23.1",
"path": "^0.12.7"
},
"keywords": [
"node",
"lambda",
"aws"
]
}Change to something like ImageResizer.js and make sure this has the same name as your Lambda function / handler
/* Dependencies: */
const async = require('async');
const AWS = require('aws-sdk');
const gm = require('gm');
const util = require('util');
const path = require('path');
const imageMagick = gm.subClass({ imageMagick: true });
const WEB_WIDTH_MAX = 420;
const WEB_Q_MAX = 85;
const FOLDER_DEST = 'thumb/';
AWS.config.update({
accessKeyId: 'xxxxxxxxxxx',
secretAccessKey: 'xxxxxxxxxxxxxxxxxxxx'
});
const s3 = new AWS.S3();
exports.handler = (event, context, callback) => {
// Read options from the event.
// console.log('Reading options from event:\n', util.inspect(event, {depth: 5}));
const srcBucket = event.Records[0].s3.bucket.name;
// Object key may have spaces or unicode non-ASCII characters.
const srcKey = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
const dstBucket = srcBucket;
const imageName = path.basename(srcKey);
// Infer the image type.
const typeMatch = srcKey.match(/\.([^.]*)$/);
if (!typeMatch) {
callback('Could not determine the image type.');
return;
}
const imageType = typeMatch[1];
if (imageType.toUpperCase() !== 'jpg'.toUpperCase() && imageType.toUpperCase() !== 'png'.toUpperCase() && imageType.toUpperCase() !== 'jpeg'.toUpperCase()) {
callback(`Unsupported image type: ${imageType}`);
return;
}
// ****************before async******************
// Download the image from S3, transform, and upload to a different S3 bucket.
async.waterfall([
function download (next) {
// Download the image from S3 into a buffer.
s3.getObject({
Bucket: srcBucket,
Key: srcKey
}, next);
},
function transformWebMax (response, next) {
imageMagick(response.Body)
.resize(WEB_WIDTH_MAX)
.quality(WEB_Q_MAX)
// .gravity('Center')
.strip()
// .crop(WEB_WIDTH_MAX, WEB_HEIGHT_MAX)
.toBuffer('jpg', (err, buffer) => {
if (err) {
console.log('An error occurred while saving IM to buffer: ', err)
return false /* stop the remaining sequence and prevent sending an empty or invalid buffer to AWS */
} else {
next(null, response, buffer)
}
})
},
function uploadWebMax (response, buffer, next) {
// Stream the transformed image to a different S3 bucket.
const dstKeyResized = FOLDER_DEST + imageName;
s3.putObject({
Bucket: dstBucket,
Key: dstKeyResized,
Body: buffer,
ContentType: response.ContentType,
CacheControl: 'max-age=864000'
}, (err, data) => {
if (err) {
console.error(err, err.stack);
} else {
console.log('uploaded to web-max Successfully !!');
next(null, response, buffer);
}
});
}
], err => {
if (err) {
console.error('Unable to resize image');
} else {
console.log('Successfully resized image');
}
callback(null, 'message');
});
};AWS Lambda offers monitoring of these functions as well as debugging (ideally you would keep all console.logs in place in order to see what is going on in case something is not working).