[SOLVED - kindof] File uploads - From Slingshot to EvaporateJS


#1

This has been doing my head in for 3ish days now, I’m hoping someone can help.

I’m trying to migrate a project to using EvaporateJS - because of the issues described here: Slingshot file upload to S3 : “error : failed to upload file to cloud storage [-0]”

I considered:
1 - Using https://github.com/VeliovGroup/Meteor-Files - but that does not seem to support direct to S3 uploads, which is important to me
2 - Forking the Slingshot project - I don’t have the technical know how to manipulate it, from the bits and pieces of the code I have looked at. At least, on the surface, migrating to EvaporateJS seems like it will require less time-resources and provide more functionality along with better community support.

However, I’m only at the early stage, trying a simple upload without anything else, but I keep running into this error: net::ERR_NAME_NOT_RESOLVED.

initiate error: <my-bucket-name>/<my-object-key-name>  status:0

The url logged by the initiate error above seems to be spot on - i.e. If the upload actually succeeded, that is where it would be located - and the files in the bucket already are successfully getting downloaded when pointing to identically constructed URL’s.

From Googling around and reading as much as I can, this seems to be an authorisation error, which leads me to believe my custom Auth function is not working as it should - specifically, I believe my function generating the V4 signature might be off.

What I have tried so far:

  • To instantiate Evaporate JS (within React):
// ...React stuff
componentWillMount() {
		// Instantiate an EvaporateJS instance
		const config = {
			customAuthMethod: (_signParams, _signHeaders, stringToSign, dateString) => {
				return new Promise((resolve, reject) => {
                                          // Meteor method call that returns a promise that resolves a V4 AWS sig
					genAwsSig.call({_signParams, _signHeaders, stringToSign, dateString}, (err, res) => {
						if (err) reject(err);
						console.log('res:', res); // This successfully logs what seems to be a legit key
						resolve(res);
					});
				})
			 },
			aws_key: 'AKIAIXJ2DK7OCYO4FBGA',
			awsRegion: 'ap-southeast-2',
			aws_url: 'https://ap-southeast-2.amazonaws.com',
			bucket: 'bucketName',
			computeContentMd5: true,
			cryptoMd5Method(data) {
				return crypto
					.createHash('md5')
					.update(data)
					.digest('base64');
			},
			cryptoHexEncodedHash256: data => AWS.util.crypto.md5(data, 'base64'),
		};
		this.evaporate = Evaporate.create(config);
	}
  • Function code within the method that generates and resolves into a V4 sig. What I have tried so far:
  1. Loosely based on the example from …example/evaporate_example.html

export function generateAWSsignature(_signParams, _signHeaders, stringToSign, dateString) {
	return new Promise((resolve, reject) => {
		try {
			const date = hmac(['AWS4', awsSecret].join(''), dateString.substr(0, 8));
			const region = hmac(date, 'ap-southeast-2');
			const service = hmac(region, 's3');
			const signing = hmac(service, 'aws4_request');
			const signingKey = AWS.util.crypto.hmac(signing, decodeURIComponent(stringToSign), 'hex');
			console.log('signing: ', signing);
			console.log('signingKey: ', signingKey);
			resolve(signingKey);
		} catch (err) {
			reject(err);
		}
	});
}

  1. Using this package - https://github.com/department-stockholm/aws-signature-v4
export function generateAWSsignature(_signParams, _signHeaders, stringToSign, dateString) {
	return new Promise((resolve, reject) => {
		try {
			resolve(createSignature(secretKey, Date.now(), 'ap-southeast-2', 's3', stringToSign));
		} catch (err) {
			reject(err);
		}
	});
}
  1. Using the way Meteor Slingshot works (current package I am using on the app) - https://github.com/CulturalMe/meteor-slingshot/
export function generateAWSsignature(_signParams, _signHeaders, stringToSign, dateString) {
	return new Promise((resolve, reject) => {
		try {
			const dateKey = hmac256("AWS4" + secretKey, dateString);
			const dateRegionKey = hmac256(dateKey, 'ap-southeast-2');
			const dateRegionServiceKey = hmac256(dateRegionKey, 's3');
			const signingKey = hmac256(dateRegionServiceKey, "aws4_request");
			resolve(hmac256(signingKey, policy, "hex")); // No idea what the 'policy' here is meant to be - so obv this did not work
		} catch (err) {
			reject(err);
		}
	});
}

So my questions:

1 - Is there something obvious I am missing in my code snippets above?
2 - What exactly does the ERR_NAME_NOT_RESOLVED message mean? Am I right in thinking it is something to do with generating the V4 signature?

Thanks in advance for any pointers/guidance anyone can provide!


Slingshot file upload to S3 : "error : failed to upload file to cloud storage [-0]"
The state of Meteor and AWS S3 file upload?
#2

Okay - here is the full working code for implementing Evaporate within Meteor. My only concern at this point (but do not think it is Meteor related based on a couple of issues in the evaporatejs repo), is that my UI more or less freezes up while an upload is happening. So fair warning, at least with the implementation below as it is, you would need to find a solution or not do background uploads like I am trying.

Initialising evaporate

import Evaporate from 'evaporate';
import AWS from 'aws-sdk'; // Really large import, optimise
import { v4sign } from '../../../../startup/api/methods/v4sign';

function customAuthMethod(_signParams, _signHeaders, stringToSign, dateString) {
	return new Promise((resolve, reject) => {
		v4sign.call({ _signParams, _signHeaders, stringToSign, dateString }, (err, signingKey) => {
			if (err) {
				reject(err);
			} else {
				resolve(signingKey);
			}
		});
	});
}

export function initialiseEvaporate() {
	const { bucket, key, region } = Meteor.settings.public.aws;
	return Evaporate.create({
		aws_key: key,
		aws_url: 'https://s3-ap-southeast-2.amazonaws.com',
		awsRegion: region,
		bucket,
		computeContentMd5: true,
		maxConcurrentParts: 5, // This is the default, I was trying to play around with this to see if I can boost performance
		cryptoMd5Method(data) {
			return AWS.util.crypto.md5(data, 'base64');
		},
		cryptoHexEncodedHash256(data) {
			return AWS.util.crypto.sha256(data, 'hex');
		},
		logging: false,
		s3FileCacheHoursAgo: 1,
		allowS3ExistenceOptimization: false,
		customAuthMethod,
		progressIntervalMS: 2000,
	});
}

v4sign.js

import { Meteor } from 'meteor/meteor';
import { SimpleSchema } from 'meteor/aldeed:simple-schema';
import { ValidatedMethod } from 'meteor/mdg:validated-method';

// Signs requests to allow direct to S3 uploads

export const v4sign = new ValidatedMethod({
	name: 'v4sign',
	validate: new SimpleSchema({
		_signParams: { type: Object },
		_signHeaders: { type: Object },
		stringToSign: { type: String },
		dateString: { type: String },
	}).validator(),
	run({ _signParams, _signHeaders, stringToSign, dateString }) {
		if (!this.userId) {
			throw new Meteor.Error('not-authorised');
		}
		let signingKey;
		if (Meteor.isServer && !this.isSimulation) {
			function hmac(k, v) {
				return AWS.util.crypto.hmac(k, v, 'buffer');
			}
			try {
				const date = hmac(['AWS4', Meteor.settings.awsSecret].join(''), dateString.substr(0, 8));
				const region = hmac(date, 'ap-southeast-2');
				const service = hmac(region, 's3');
				const signing = hmac(service, 'aws4_request');
				signingKey = AWS.util.crypto.hmac(signing, decodeURIComponent(stringToSign), 'hex');
			} catch (err) {
				handleError('Error creating v4 sig', err);
			}
    }
    return signingKey;
	},
});

Within your react component

// Get your file object(s) plus other req info
files.forEach(file => {
			const { status, fileObject, s3Key } = file;
			if (status === 'pending') {
				this.evaporate.then(ev => {
					ev
						.add({
							name: s3Key,
							file: fileObject,
							progress(a, stats) {
								updateFileProp('uploadStats', s3Key, stats);
							},
						})
						.then(key => {
							console.log(`Uploaded successfully: Key: ${key}`);
							updateFileProp('status', s3Key, 'complete');
						})
						.catch(err => console.error(err));
					updateFileProp('status', s3Key, 'uploading');
				});
			}
		});

If anybody has any ideas regarding performance (with the freezing UI), then I would very much appreciate them!