How to determine if object exists AWS S3 Node.JS sdk
node.jsAmazon Web-ServicesAmazon S3node.js Problem Overview
I need to check if a file exists using AWS SDK. Here is what I'm doing:
var params = {
Bucket: config.get('s3bucket'),
Key: path
};
s3.getSignedUrl('getObject', params, callback);
It works but the problem is that when the object doesn't exists, the callback (with arguments err and url) returns no error, and when I try to access the URL, it says "NoSuchObject".
Shouldn't this getSignedUrl method return an error object when the object doesn't exists? How do I determine if the object exists? Do I really need to make a call on the returned URL?
node.js Solutions
Solution 1 - node.js
Before creating the signed URL, you need to check if the file exists directly from the bucket. One way to do that is by requesting the HEAD metadata.
// Using callbacks
s3.headObject(params, function (err, metadata) {
if (err && err.name === 'NotFound') {
// Handle no object on cloud here
} else if (err) {
// Handle other errors here....
} else {
s3.getSignedUrl('getObject', params, callback);
// Do stuff with signedUrl
}
});
// Using async/await
try {
await s3.headObject(params).promise();
const signedUrl = s3.getSignedUrl('getObject', params);
// Do stuff with signedUrl
} catch (error) {
if (error.name === 'NotFound') {
// Handle no object on cloud here...
} else {
// Handle other errors here....
}
}
Solution 2 - node.js
The simplest solution without try/catch block.
const exists = await s3
.headObject({
Bucket: S3_BUCKET_NAME,
Key: s3Key,
})
.promise()
.then(
() => true,
err => {
if (err.code === 'NotFound') {
return false;
}
throw err;
}
);
Solution 3 - node.js
by using headObject method
AWS.config.update({
accessKeyId: "*****",
secretAccessKey: "****",
region: region,
version: "****"
});
const s3 = new AWS.S3();
const params = {
Bucket: s3BucketName,
Key: "filename" //if any sub folder-> path/of/the/folder.ext
}
try {
await s3.headObject(params).promise()
console.log("File Found in S3")
} catch (err) {
console.log("File not Found ERROR : " + err.code)
}
As params are constant, the best way to use it with const. If the file is not found in the s3 it throws the error NotFound : null.
If you want to apply any operations in the bucket, you have to change the permissions of CORS Configuration in the respective bucket in the AWS. For changing permissions Bucket->permission->CORS Configuration and Add this code.
<CORSConfiguration>
<CORSRule>
<AllowedOrigin>*</AllowedOrigin>
<AllowedMethod>PUT</AllowedMethod>
<AllowedMethod>POST</AllowedMethod>
<AllowedMethod>DELETE</AllowedMethod>
<AllowedMethod>GET</AllowedMethod>
<AllowedMethod>HEAD</AllowedMethod>
<AllowedHeader>*</AllowedHeader>
</CORSRule>
</CORSConfiguration>
for more information about CORS Configuration : https://docs.aws.amazon.com/AmazonS3/latest/dev/cors.html
Solution 4 - node.js
Use getObject method like this:
var params = {
Bucket: config.get('s3bucket'),
Key: path
};
s3.getObject(params, function(err, data){
if(err) {
console.log(err);
}else {
var signedURL = s3.getSignedUrl('getObject', params, callback);
console.log(signedURL);
}
});
Solution 5 - node.js
You can also use the waitFor method together with the state objectExists. This will use S3.headObject() internally.
var params = {
Bucket: config.get('s3bucket'),
Key: path
};
s3.waitFor('objectExists', params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data); // successful response
});
Solution 6 - node.js
> Promise.All without failure Synchronous Operation
var request = require("request");
var AWS = require("aws-sdk");
AWS.config.update({
accessKeyId: "*******",
secretAccessKey: "***********"
});
const s3 = new AWS.S3();
var response;
function initialize(bucket,key) {
// Setting URL and headers for request
const params = {
Bucket: bucket,
Key: key
};
// Return new promise
return new Promise(function(resolve, reject) {
s3.headObject(params, function(err, resp, body) {
if (err) {
resolve(key+"/notfound");
} else{
resolve(key+"/found");
}
})
})
}
function main() {
var foundArray = new Array();
var notFoundArray = new Array();
var prefix = 'abc/test/';
var promiseArray = [];
try{
for(var i=0;i<10;i++)
{
var key = prefix +'1234' + i;
console.log("Key : "+ key);
promiseArray[i] = initialize('bucket',key);
promiseArray[i].then(function(result) {
console.log("Result : " + result);
var temp = result.split("/");
console.log("Temp :"+ temp);
if (temp[3] === "notfound")
{
console.log("NOT FOUND");
}else{
console.log("FOUND");
}
}, function(err) {
console.log (" Error ");
});
}
Promise.all(promiseArray).then(function(values) {
console.log("^^^^^^^^^^^^TESTING****");
}).catch(function(error) {
console.error(" Errro : "+ error);
});
}catch(err){
console.log(err);
}
}
main();
Solution 7 - node.js
> Synchronous call on S3 in Nodejs instead of asynchronous call using Promise
var request = require("request");
var AWS = require("aws-sdk");
AWS.config.update({
accessKeyId: "*****",
secretAccessKey: "********"
});
const s3 = new AWS.S3();
var response;
function initialize(bucket,key) {
// Setting URL and headers for request
const params = {
Bucket: bucket,
Key: key
};
// Return new promise
return new Promise(function(resolve, reject) {
s3.headObject(params, function(err, resp, body) {
if (err) {
console.log('Not Found : ' + params.Key );
reject(params.Key);
} else {
console.log('Found : ' + params.Key );
resolve(params.Key);
}
})
})
}
function main() {
var foundArray = new Array();
var notFoundArray = new Array();
for(var i=0;i<10;i++)
{
var key = '1234'+ i;
var initializePromise = initialize('****',key);
initializePromise.then(function(result) {
console.log('Passed for : ' + result);
foundArray.push(result);
console.log (" Found Array : "+ foundArray);
}, function(err) {
console.log('Failed for : ' + err);
notFoundArray.push(err);
console.log (" Not Found Array : "+ notFoundArray);
});
}
}
main();
Solution 8 - node.js
> Synchronous Put Operation
var request = require("request");
var AWS = require("aws-sdk");
AWS.config.update({
accessKeyId: "*****",
secretAccessKey: "***"
});
const s3 = new AWS.S3();
var response;
function initialize(bucket,key) {
// Setting URL and headers for request
const params = {
Bucket: bucket,
Key: key
};
// Return new promise
return new Promise(function(resolve, reject) {
s3.putObject(params, function(err, resp, body) {
if (err) {
reject();
} else {
resolve();
}
})
})
}
function main() {
var promiseArray = [];
var prefix = 'abc/test/';
for(var i=0;i<10;i++)
{
var key = prefix +'1234'+ i;
promiseArray[i] = initialize('bucket',key);
promiseArray[i].then(function(result) {
console.log (" Successful ");
}, function(err) {
console.log (" Error ");
});
}
console.log('Promises ' + promiseArray);
Promise.all(promiseArray).then(function(values) {
console.log("******TESTING****");
});
}
main();
Solution 9 - node.js
AFAICT the correct way to do this as of February 2022, with JavaScript V3 SDK is
to use the HeadObjectCommand.
Note: I'm using TypeScript here with explicit typings, but you can remove those explicit typings when you refactor the code...they're just to show the AWS types in use.
import {
S3Client,
HeadObjectCommand, HeadObjectCommandInput, HeadObjectCommandOutput,
} from '@aws-sdk/client-s3';
function async existsInS3(
client: S3Client,
bucket: string,
key: string,
): Promise<boolean> {
try {
const bucketParams: HeadObjectCommandInput = {
Bucket: bucket,
Key: key,
};
const cmd = new HeadObjectCommand(bucketParams);
const data: HeadObjectCommandOutput = await client.send(cmd);
// I always get 200 for my testing if the object exists
const exists = data.$metadata.httpStatusCode === 200;
return exists;
} catch (error) {
if (error.$metadata?.httpStatusCode === 404) {
// doesn't exist and permission policy includes s3:ListBucket
return false;
} else if (error.$metadata?.httpStatusCode === 403) {
// doesn't exist, permission policy WITHOUT s3:ListBucket
return false;
} else {
// some other error
...log and rethrow if you like
}
}
}
If you look in the Permissions section of the HeadObjectCommand documentation linked above, you'll notice
it mentions the 403 and 404 responses:
> You need the relevant read object (or version) permission for this operation. For more information, see Specifying Permissions in a Policy. If the object you request does not exist, the error Amazon S3 returns depends on whether you also have the s3:ListBucket permission. > > If you have the s3:ListBucket permission on the bucket, Amazon S3 returns an HTTP status code 404 ("no such key") error. > > If you don’t have the s3:ListBucket permission, Amazon S3 returns an HTTP status code 403 ("access denied") error.
I don't know if these error responses could stem from other errors in addition to the non-existence of the key.
CORS
I also had to add HEAD in the AllowedMethods section of CORS permissions on the bucket:
"AllowedMethods": [
"GET",
"PUT",
"HEAD"
],