@gulpetl/vinyl-s3

Use S3 as a source or destination of vinyl files. Forked from vinyl-s3 to add support for Node-RED as part of the gulpetl project

Features:

• Source with multi-globbing support,
• Use either streaming or buffering,
• Upload or download files,
• Pass custom options to S3,
• Smart Content-Type and Content-Encoding detection,
• Works great with gulp.

Usage

var gulp = require('gulp'),
	s3 = require('vinyl-s3');

// Upload files to S3
gulp.task('upload', function() {
	return gulp.src('data/*.jpg', { buffer: false })
		.pipe(s3.dest('s3://my-bucket/prefix'));
});

// Download files from S3
gulp.task('download', function() {
	return s3.src('s3://my-bucket/prefix/*.jpg', { buffer: false })
		.pipe(gulp.dest('data'));
});

// Just print a list of files
var through2 = require('through2');
gulp.task('meta', function() {
	return s3.src('s3://my-bucket/foo/**/*.jpg', { read: false })
		.pipe(through2.obj(function(file, _, callback) {
			console.log(file.path);
			callback();
		}));
});

When working with large files you may find it useful to use streaming mode instead of buffering mode. You can enable this in the src() family of functions by setting { buffer: false }. The default mode is to use buffering as is the same with fs.src.

src

See getObject for a list of supported options.

// Specify custom attributes via S3 URL.
s3.src('s3://bucket/key/*?IfModifiedSince=123456789')
    .pipe(fs.dest('downloads'));

// Specify custom attributes by passing in an AWS options object.
s3.src({
    Bucket: 'bucket',
    Key: 'key/*',
    IfModifiedSince: Date.now()
}).pipe(fs.dest('downloads'));

// Use multiple source buckets and patterns.
s3.src(['s3://bucket1/*.jpg', 's3://bucket1/*.png', 's3://bucket2/*.gif'])
    .pipe(fs.dest('downloads'));

dest

See putObject and upload for a list of supported options. There is limited support for automatically detecting the correct Content-Type and correct Content-Encoding. Parallel uploads are supported by passing { queueSize: n } as a second parameter.

// Specify custom attributes via S3 URL.
fs.src('files/*.jpg')
    .pipe(s3.dest('s3://bucket/foo?ContentType=image/jpeg'));

// Specify custom attributes by passing in an AWS options object.
fs.src('files/*.jpg')
    .pipe(s3.dest({
        Bucket: 'bucket',
        Key: 'foo',
        ContentType: 'image/jpeg'
    }));

// Specify custom attributes per file.
fs.src('files/*.jpg')
    .pipe(through2.obj(function(file, enc, next) {
        // There are some non-standard properties on the file object that
        // are used to generate certain AWS options.
        file.contentType = 'image/jpeg';
        file.contentEncoding = 'gzip';

        // Setting the awsOptions property on a file causes the object to be
        // included in the command sent to S3. These options override any
        // previously set value.
        file.awsOptions = {
            ACL: 'private',
            CacheControl: 'max-age=1296000',
            ContentType: 'image/jpeg',
            Metadata: {
                color: 'red'
            }
        };
        this.push(file);
        next();
    }))
    .pipe(s3.dest('s3://bucket/foo'));

Node-RED

Node-RED is a low-code, visual programming environment for event-driven applications. vinyl-s3 is available there as part of the gulp-etl project; to install it to Node-RED under Manage Palette search for vinyl-s3

Demo flow

Copy this flow for import in Node-RED under Import:

[{"id":"ea8a01833caca064","type":"inject","z":"900570325b249093","name":"click to start","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"",
"payload":"","payloadType":"date","x":850,"y":40,"wires":[["581a22165aef5763"]]},{"id":"34e02d4be11bc045","type":"gulp.dest","z":"900570325b249093","name":"","path":"data","x":1210,"y":160,"wires":[["9d4b2a68ebec230e"]]},
{"id":"9d4b2a68ebec230e","type":"debug","z":"900570325b249093","name":"debug","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto",
"x":1290,"y":80,"wires":[]},{"id":"f03409d378908dad","type":"s3.src","z":"900570325b249093","name":"","path":"s3://my-bucket/prefix/*.jpg","x":970,"y":160,"wires":[["34e02d4be11bc045"]]},{"id":"e3f3197743972a9b","type":"comment",
"z":"900570325b249093","name":"Download files from S3","info":"","x":1100,"y":60,"wires":[]},{"id":"581a22165aef5763","type":"template","z":"900570325b249093","name":"Set config","field":"config","fieldType":"msg",
"format":"json","syntax":"plain","template":"{\n    \"buffer\": false,\n    \"s3\": {\n        \"accessKeyId\": \"AWS_ACCESS_ID\",\n        \"secretAccessKey\": \"AWS_ACCESS_KEY\",\n        \"region\": \"us-west-2\"\n    }\n}",
"output":"json","x":880,"y":100,"wires":[["f03409d378908dad"]]}]