How do I ungzip (decompress) a NodeJS request's module gzip response body?
Javascriptnode.jsExpressZlibJavascript Problem Overview
How do I unzip a gzipped body in a request's module response?
I have tried several examples around the web but none of them appear to work.
request(url, function(err, response, body) {
if(err) {
handleError(err)
} else {
if(response.headers['content-encoding'] == 'gzip') {
// How can I unzip the gzipped string body variable?
// For instance, this url:
// http://highsnobiety.com/2012/08/25/norse-projects-fall-2012-lookbook/
// Throws error:
// { [Error: incorrect header check] errno: -3, code: 'Z_DATA_ERROR' }
// Yet, browser displays page fine and debugger shows its gzipped
// And unzipped by browser fine...
if(response.headers['content-encoding'] && response.headers['content-encoding'].toLowerCase().indexOf('gzip') > -1) {
var body = response.body;
zlib.gunzip(response.body, function(error, data) {
if(!error) {
response.body = data.toString();
} else {
console.log('Error unzipping:');
console.log(error);
response.body = body;
}
});
}
}
}
}
Javascript Solutions
Solution 1 - Javascript
I couldn't get request to work either, so ended up using http instead.
var http = require("http"),
zlib = require("zlib");
function getGzipped(url, callback) {
// buffer to store the streamed decompression
var buffer = [];
http.get(url, function(res) {
// pipe the response into the gunzip to decompress
var gunzip = zlib.createGunzip();
res.pipe(gunzip);
gunzip.on('data', function(data) {
// decompression chunk ready, add it to the buffer
buffer.push(data.toString())
}).on("end", function() {
// response and decompression complete, join the buffer and return
callback(null, buffer.join(""));
}).on("error", function(e) {
callback(e);
})
}).on('error', function(e) {
callback(e)
});
}
getGzipped(url, function(err, data) {
console.log(data);
});
Solution 2 - Javascript
try adding encoding: null
to the options you pass to request
, this will avoid converting the downloaded body to a string and keep it in a binary buffer.
Solution 3 - Javascript
Like @Iftah said, set encoding: null
.
Full example (less error handling):
request = require('request');
zlib = require('zlib');
request(url, {encoding: null}, function(err, response, body){
if(response.headers['content-encoding'] == 'gzip'){
zlib.gunzip(body, function(err, dezipped) {
callback(dezipped.toString());
});
} else {
callback(body);
}
});
Solution 4 - Javascript
Actually request module handles the gzip response. In order to tell the request module to decode the body argument in the callback function, We have to set the 'gzip' to true in the options. Let me explain you with an example.
Example:
var opts = {
uri: 'some uri which return gzip data',
gzip: true
}
request(opts, function (err, res, body) {
// now body and res.body both will contain decoded content.
})
Note: The data you get on 'reponse' event is not decoded.
This works for me. Hope it works for you guys too.
The similar problem usually we ran into while working with request module is with JSON parsing. Let me explain it. If u want request module to automatically parse the body and provide you JSON content in the body argument. Then you have to set 'json' to true in the options.
var opts = {
uri:'some uri that provides json data',
json: true
}
request(opts, function (err, res, body) {
// body and res.body will contain json content
})
Reference: https://www.npmjs.com/package/request#requestoptions-callback
Solution 5 - Javascript
As seen in https://gist.github.com/miguelmota/9946206:
Both request and request-promise handle it out of the box as of Dec 2017:
var request = require('request')
request(
{ method: 'GET'
, uri: 'http://www.google.com'
, gzip: true
}
, function (error, response, body) {
// body is the decompressed response body
console.log('server encoded the data as: ' + (response.headers['content-encoding'] || 'identity'))
console.log('the decoded data is: ' + body)
}
)
Solution 6 - Javascript
I have formulated a more complete answer after trying the different ways to gunzip, and solving errors to do with encoding.
Hope this helps you too:
var request = require('request');
var zlib = require('zlib');
var options = {
url: 'http://some.endpoint.com/api/',
headers: {
'X-some-headers' : 'Some headers',
'Accept-Encoding' : 'gzip, deflate',
},
encoding: null
};
request.get(options, function (error, response, body) {
if (!error && response.statusCode == 200) {
// If response is gzip, unzip first
var encoding = response.headers['content-encoding']
if (encoding && encoding.indexOf('gzip') >= 0) {
zlib.gunzip(body, function(err, dezipped) {
var json_string = dezipped.toString('utf-8');
var json = JSON.parse(json_string);
// Process the json..
});
} else {
// Response is not gzipped
}
}
});
Solution 7 - Javascript
Here is my two cents worth. I had the same problem and found a cool library called concat-stream
:
let request = require('request');
const zlib = require('zlib');
const concat = require('concat-stream');
request(url)
.pipe(zlib.createGunzip())
.pipe(concat(stringBuffer => {
console.log(stringBuffer.toString());
}));
Solution 8 - Javascript
Here's a working example (using the request module for node) that gunzips the response
function gunzipJSON(response){
var gunzip = zlib.createGunzip();
var json = "";
gunzip.on('data', function(data){
json += data.toString();
});
gunzip.on('end', function(){
parseJSON(json);
});
response.pipe(gunzip);
}
Full code: https://gist.github.com/0xPr0xy/5002984
Solution 9 - Javascript
With got
, a request
alternative, you can simply do:
got(url).then(response => {
console.log(response.body);
});
Decompression is handled automagically when needed.
Solution 10 - Javascript
I'm using node-fetch. I was getting response.body
, what I really wanted was await response.text()
.