Skip to content

simpler implementation using async/await #6

@roeycohen

Description

@roeycohen

I've create an implementation which i found more concise and easier to understand (using async/await where possible),
i hope someone will find it useful :)

const S3Multipart = async function ({
	file, //File to upload
	createUpload, //async function(totalParts) { ... }
	getPartUrl, //async function(partNumber) { ... }
	completeUpload, //async function(etags) { ... }
	parallelism = 3,
	partSize = 100 * 1024 * 1024,
	onProgress, //function(uploadedBytes, totalBytes) { ... }
	retries = 3,
		retryBackoffTimeMs = retry => retry * retry * 1000,
} = {})
{
	const totalParts = Math.ceil(file.size / partSize);

	await createUpload(totalParts);

	const etags = new Array(totalParts);
	let activeXhr = [], partNumber = 0, completedParts = 0, totalSentBytes = 0;

	try
	{
		await new Promise((resolve, reject) =>
		{
			const nextPart = () =>
			{
				if (partNumber >= totalParts)
				{
					if (completedParts === totalParts)
						return resolve();
				}
				else
				{
					const currentPartNumber = ++partNumber;
					let attempt = 0;
					const tryUpload = async () =>
					{
						let xhr;
						try
						{
							const partStart = (partNumber - 1) * partSize;
							const part = file.slice(partStart, partStart + partSize);
							const partUrl = await getPartUrl(partNumber);
							if (!partUrl)
								throw new Error("No URL for part " + partNumber, {cause: 'noRetry'});

							xhr = new XMLHttpRequest();
							activeXhr.push(xhr);
							const etag = await new Promise((resolve, reject) =>
							{
								xhr.onreadystatechange = () =>
								{
									if (xhr.readyState === 4)
									{
										if (xhr.status === 200)
											resolve(xhr.getResponseHeader("ETag"));
										else
											reject(new Error(`Unexpected response HTTP ${xhr.status} ${xhr.statusText}`));
									}
								};

								let lastLoaded = 0;
								if (onProgress)
								{
									xhr.upload.onprogress = e =>
									{
										totalSentBytes += e.loaded - lastLoaded;
										lastLoaded = e.loaded;
										onProgress(totalSentBytes, file.size);
									};
								}

								xhr.open("PUT", partUrl, true);
								xhr.setRequestHeader("Content-Type", "");
								xhr.send(part);
							});

							etags[currentPartNumber - 1] = etag;
							completedParts++;
							nextPart();
						}
						catch (err)
						{
							if (err.cause !== 'noRetry' && attempt++ < retries)
							{
								const delay = typeof retryBackoffTimeMs === "function" ? retryBackoffTimeMs(attempt) : retryBackoffTimeMs;
								setTimeout(tryUpload, delay);
							}
							else
							{
								reject(err);
							}
						}
						finally
						{
							activeXhr = activeXhr.filter(x => x !== xhr);
						}
					};
					tryUpload();
				}
			};

			for (let i = 0; i < parallelism; i++)
				nextPart();
		});
		await completeUpload(etags);
	}
	catch (err)
	{
		activeXhr.forEach(xhr => xhr.abort());
		throw err;
	}
}

calling the function should look like this:
(file & uploadId in events were left out, as we need file key anyway and this data doesn't need to be maintained inside the upload function)

try
{

	await S3Multipart({
		file,
		createUpload: async (totalParts) =>
		{
                        //totalParts can help creating enough presigned urls a head
			/*...*/
		},
		getPartUrl: async (partNumber) =>
		{
			/*...*/
		},
		completeUpload: async (etags) =>
		{
			/*...*/
		},
		onProgress: (uploadedBytes, totalBytes) => console.log('progress', 100 * uploadedBytes / totalBytes + '%')
	});
	console.log("Upload complete successfully";
}
catch (err)
{
	console.log("Upload failed: " + err);
}

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions