stream-multipart-upload
v0.4.6
Published
upload multipart/form-data files in a stream manner
Downloads
14
Readme
stream-multipart-upload
Features
Install
npm install stream-multipart-upload
Example
const server = http.createServer((req, res) => {
const multipart = req.pipe(new Multipart({ headers: req.headers }));
const zip = new Zip();
const FS_UPLOADS_DIR = '/var/www/uploads';
const S3_BUCKET = 'test-stream-multipart-upload';
const S3_UPLOADS_PATH = 'uploads';
const S3_TEMP_PATH = 'temp';
multipart.once('data', () => {
res.writeHead(200, { 'Content-Type': 'application/json' });
});
multipart.once('error', (err) => {
if (!res.headersSent) {
res.writeHead(400, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ error: err.message }));
}
});
multipart.pipe(new MultipartError()).pipe(zip);
multipart.pipe(new Pick('metadata')).pipe(zip);
multipart.pipe(new FileSize()).pipe(zip);
multipart.pipe(new FileHash({ encoding: 'bs58' })).pipe(zip);
multipart.pipe(new StorageTempLocal()).pipe(zip);
multipart.pipe(new StorageTempS3({ bucket: S3_BUCKET, path: S3_TEMP_PATH })).pipe(zip);
multipart.pipe(new Exiftool()).pipe(zip);
zip
.pipe(new Merge())
.pipe(new StorageLocal({ dir: FS_UPLOADS_DIR }))
.pipe(new StorageS3({ bucket: S3_BUCKET, path: S3_UPLOADS_PATH, saveMetadata: true }))
.pipe(new StringifyError())
.pipe(new JsonStream())
.pipe(res);
});
API
Performance
Hardware
KVM, 1 vCPU Intel Xeon E5-2670v2, 2GB RAM, SSD
Local server, local client, 30 DSLR photos with average size of 10Mb each
task: concurrency=3, files-per-request=10
requests: 3
files: 30
size: 314.42 Mb
speed: 891.96 Mbit/s, 10.64 files per second
Local server, local client, 6610 exiftool sample images
task: concurrency=10, files-per-request=10
requests: 661
files: 6610
size: 181.34 Mb
speed: 22.81 Mbit/s, 103.93 files per second
cpu usage: 100% (40% node process, 60% exiftool process)