Save cURL content result into a string in C++

C++ Problem Overview

int main(void)
{
  CURL *curl;
  CURLcode res;
 
  curl = curl_easy_init();
  if(curl) {
    curl_easy_setopt(curl, CURLOPT_URL, "http://www.google.com");
    curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
    curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
    res = curl_easy_perform(curl);
    curl_easy_cleanup(curl);
  }
  _getch();
  return 0;
}

string contents = "";

I would like to save the result of the curl html content in a string, how do I do this? It's a silly question but unfortunately, I couldn't find anywhere in the cURL examples for C++ thanks!

C++ Solutions

Solution 1 - C++

You will have to use http://curl.haxx.se/libcurl/c/getinmemory.html">`CURLOPT_WRITEFUNCTION`</a> to set a callback for writing. I can't test to compile this right now, but the function should look something close to;

static std::string readBuffer;

static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp)
{ 
    size_t realsize = size * nmemb;
    readBuffer.append(contents, realsize);
    return realsize;
}

Then call it by doing;

readBuffer.clear();
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
// ...other curl options
res = curl_easy_perform(curl);

After the call, readBuffershould have your contents.

Edit: You can use CURLOPT_WRITEDATA to pass the buffer string instead of making it static. In this case I just made it static for simplicity. A good page to look (besides the linked example above) is http://curl.haxx.se/libcurl/c/curl_easy_setopt.html">here</a> for an explanation of the options.

Edit2: As requested, here's a complete working example without the static string buffer;

#include <iostream>
#include <string>
#include <curl/curl.h>


static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp)
{
    ((std::string*)userp)->append((char*)contents, size * nmemb);
    return size * nmemb;
}

int main(void)
{
  CURL *curl;
  CURLcode res;
  std::string readBuffer;

  curl = curl_easy_init();
  if(curl) {
    curl_easy_setopt(curl, CURLOPT_URL, "http://www.google.com");
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
    res = curl_easy_perform(curl);
    curl_easy_cleanup(curl);

    std::cout << readBuffer << std::endl;
  }
  return 0;
}

Solution 2 - C++

Using the 'new' C++11 lambda functionality, this can be done in a few lines of code.

#ifndef WIN32 #define __stdcall "" #endif //For compatibility with both Linux and Windows
std::string resultBody { };
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &resultBody);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, static_cast<size_t (__stdcall *)(char*, size_t, size_t, void*)>(
    [](char* ptr, size_t size, size_t nmemb, void* resultBody){
	    *(static_cast<std::string*>(resultBody)) += std::string {ptr, size * nmemb};
		return size * nmemb;
	}
));

CURLcode curlResult = curl_easy_perform(curl);
std::cout << "RESULT BODY:\n" << resultBody << std::endl;
// Cleanup etc

Note the __stdcall cast is needed to comply to the C calling convention (cURL is a C library)

Solution 3 - C++

This might not work right away but should give you an idea:

#include <string>
#include <curl.h>
#include <stdio.h>
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
    size_t written;
    written = fwrite(ptr, size, nmemb, stream);
    return written;
}
 
int main() {
    std::string tempname = "temp";
    CURL *curl;
    CURLcode res;
    curl = curl_easy_init();
	if(curl) {
	  FILE *fp = fopen(tempname.c_str(),"wb");
	  curl_easy_setopt(curl, CURLOPT_URL, "http://www.google.com");
	  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data); 
	  curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
	  res = curl_easy_perform(curl);
	  curl_easy_cleanup(curl);
      fclose(fp);
      fp = fopen(tempname.c_str(),"rb");
	  fseek (fp , 0 , SEEK_END);
	  long lSize = ftell (fp);
	  rewind(fp);
      char *buffer = new char[lSize+1];
	  fread (buffer, 1, lSize, fp);
	  buffer[lSize] = 0;
	  fclose(fp);
      std::string content(buffer);
      delete [] buffer;
	}
}

Solution 4 - C++

On my blog I have published a simple wrapper class to perform this task.

Usage example:

#include "HTTPDownloader.hpp"

int main(int argc, char** argv) {
    HTTPDownloader downloader;
    std::string content = downloader.download("https://stackoverflow.com");
    std::cout << content << std::endl;
}

Here's the header file:

/**
 * HTTPDownloader.hpp
 *
 * A simple C++ wrapper for the libcurl easy API.
 *
 * Written by Uli Köhler (techoverflow.net)
 * Published under CC0 1.0 Universal (public domain)
 */
#ifndef HTTPDOWNLOADER_HPP
#define HTTPDOWNLOADER_HPP

#include <string>

/**
 * A non-threadsafe simple libcURL-easy based HTTP downloader
 */
class HTTPDownloader {
public:
    HTTPDownloader();
    ~HTTPDownloader();
    /**
     * Download a file using HTTP GET and store in in a std::string
     * @param url The URL to download
     * @return The download result
     */
    std::string download(const std::string& url);
private:
    void* curl;
};

#endif  /* HTTPDOWNLOADER_HPP */

Here's the source code:

/**
 * HTTPDownloader.cpp
 *
 * A simple C++ wrapper for the libcurl easy API.
 *
 * Written by Uli Köhler (techoverflow.net)
 * Published under CC0 1.0 Universal (public domain)
 */
#include "HTTPDownloader.hpp"
#include <curl/curl.h>
#include <curl/easy.h>
#include <curl/curlbuild.h>
#include <sstream>
#include <iostream>
using namespace std;

size_t write_data(void *ptr, size_t size, size_t nmemb, void *stream) {
    string data((const char*) ptr, (size_t) size * nmemb);
    *((stringstream*) stream) << data;
    return size * nmemb;
}

HTTPDownloader::HTTPDownloader() {
    curl = curl_easy_init();
}

HTTPDownloader::~HTTPDownloader() {
    curl_easy_cleanup(curl);
}

string HTTPDownloader::download(const std::string& url) {
    curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
    /* example.com is redirected, so we tell libcurl to follow redirection */
    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
    curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); //Prevent "longjmp causes uninitialized stack frame" bug
    curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "deflate");
    std::stringstream out;
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &out);
    /* Perform the request, res will get the return code */
    CURLcode res = curl_easy_perform(curl);
    /* Check for errors */
    if (res != CURLE_OK) {
        fprintf(stderr, "curl_easy_perform() failed: %s\n",
                curl_easy_strerror(res));
    }
    return out.str();
}

Solution 5 - C++

Came out with useful, yet simple solution, which overloads std::ostream::operator<<

#include <ostream>

#include <curl/curl.h>

size_t curlCbToStream (
	char * buffer,
	size_t nitems,
	size_t size,
	std::ostream * sout
)
{
	*sout << buffer;

	return nitems * size;
}

std::ostream & operator<< (
	std::ostream & sout,
	CURL * request
)
{
	::curl_easy_setopt(request, CURLOPT_WRITEDATA, & sout);
	::curl_easy_setopt(request, CURLOPT_WRITEFUNCTION, curlCbToStream);
	::curl_easy_perform(request);

	return sout;
}

Possible drawback of taken approach could be:

typedef void CURL;

That means it covers all known pointer types.

Solution 6 - C++

Based on @JoachimIsaksson answer, here is a more verbose output that handles out-of-memory and has a limit for the maximum output from curl (as CURLOPT_MAXFILESIZE limits only based on header information and not on the actual size transferred ).

#DEFINE MAX_FILE_SIZE = 10485760 //10 MiB

size_t curl_to_string(void *ptr, size_t size, size_t count, void *stream)
{
	if(((string*)stream)->size() + (size * count) > MAX_FILE_SIZE)
	{
		cerr<<endl<<"Could not allocate curl to string, output size (current_size:"<<((string*)stream)->size()<<"bytes + buffer:"<<(size * count) << "bytes) would exceed the MAX_FILE_SIZE ("<<MAX_FILE_SIZE<<"bytes)";
		return 0;
	}
	int retry=0;
	while(true)
	{
		try{
			((string*)stream)->append((char*)ptr, 0, size*count);
			break;// successful
		}catch (const std::bad_alloc&) {
			retry++;
			if(retry>100)
			{
				cerr<<endl<<"Could not allocate curl to string, probably not enough memory, aborting after : "<<retry<<" tries at 10s apart";
				return 0;
			}
			cerr<<endl<<"Could not allocate curl to string, probably not enough memory, sleeping 10s, try:"<<retry;
			sleep(10);
		}
	}
  return size*count;
}

Solution 7 - C++

I use Joachim Isaksson's answer with a modern C++ adaptation of CURLOPT_WRITEFUNCTION.

No nagging by the compiler for C-style casts.

static auto WriteCallback(char* ptr, size_t size, size_t nmemb, void* userdata) -> size_t {
  static_cast<string*>(userdata)->append(ptr, size * nmemb);
  return size * nmemb;
}

Content Type	Original Author	Original Content on Stackoverflow
Question	Grego	View Question on Stackoverflow
Solution 1 - C++	Joachim Isaksson	View Answer on Stackoverflow
Solution 2 - C++	Mark Laagland	View Answer on Stackoverflow
Solution 3 - C++	perreal	View Answer on Stackoverflow
Solution 4 - C++	Uli Köhler	View Answer on Stackoverflow
Solution 5 - C++	Jakub Koszuliński	View Answer on Stackoverflow
Solution 6 - C++	Stefan Rogin	View Answer on Stackoverflow
Solution 7 - C++	Simog	View Answer on Stackoverflow