src/server/proxy/httpclient.c

/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 2026 Olaf Wintermann. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *   1. Redistributions of source code must retain the above copyright
 *      notice, this list of conditions and the following disclaimer.
 *
 *   2. Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in the
 *      documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include "httpclient.h"

#include "../util/socket.h"

#include <cx/buffer.h>
#include <cx/string.h>
#include <cx/printf.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>

static int client_connected(EventHandler *ev, Event *event);
static int client_io(EventHandler *ev, Event *event);
static int client_process(HttpClient *client, Event *event);
static int client_finished(EventHandler *ev, Event *event);

static int client_send_buf(HttpClient *client);
static int client_send_request_body(HttpClient *client);
static int client_read_response_header(HttpClient *client);
static int client_read_response_body(HttpClient *client);

HttpClient* http_client_new(EventHandler *ev) {
    CxMempool *mp = cxMempoolCreate(32, CX_MEMPOOL_TYPE_PURE);
    if(!mp) {
        return NULL;
    }
    
    HttpClient *client = malloc(sizeof(HttpClient));
    HeaderArray *req_headers = header_array_create();
    HeaderArray *resp_headers = header_array_create();
    if(!client || !req_headers || !resp_headers) {
        free(client);
        header_array_free(req_headers);
        header_array_free(resp_headers);
        cxMempoolFree(mp);
        return NULL;
    }
    
    memset(client, 0, sizeof(HttpClient));
    client->ev = ev;
    client->socketfd = -1;
    client->request_headers = req_headers;
    client->response_headers = resp_headers;
    
    client->buffer.maxsize = HTTP_CLIENT_BUFFER_SIZE;
    client->buffer.inbuf = malloc(HTTP_CLIENT_BUFFER_SIZE);
    HttpParser *parser = http_parser_new2(1, &client->buffer, resp_headers);
    if(!parser || !client->buffer.inbuf) {
        http_client_free(client);
        return NULL;
    }
    client->parser = parser;
    
    return client;
}

void http_client_free(HttpClient *client) {
    cxMempoolFree(client->mp);
    header_array_free(client->request_headers);
    http_parser_free(client->parser);
    if(client->stream) {
        client->stream->st.free(&client->stream->st);
    }
    free(client->buffer.inbuf);
    free(client->transfer_buffer);
    free(client->transfer2_buffer);
    free(client->addr);
    free(client->method);
    free(client->uri);
    free(client);
}

int http_client_set_addr(HttpClient *client, int domain, const struct sockaddr *addr, socklen_t addrlen) {
   free(client->addr);
   client->addr = NULL;
   client->addrlen = 0;
   client->domain = domain;
   
   void *newaddr = malloc(addrlen);
   if(!newaddr) {
       return 1;
   }
   memcpy(newaddr, addr, addrlen);
   client->addr = newaddr;
   client->addrlen = addrlen;
   
   return 0;
}

int http_client_set_socket(HttpClient *client, int socketfd) {
    client->socketfd = socketfd;
    if(util_socket_setnonblock(socketfd, 1)) {
        client->socketfd = -1;
        return 1;
    }
    return 0;
}

int http_client_set_method(HttpClient *client, const char *method) {
    return http_client_set_method_len(client, method, method ? strlen(method) : 0);
}

int http_client_set_uri(HttpClient *client, const char *uri) {
    return http_client_set_uri_len(client, uri, uri ? strlen(uri) : 0);
}

static int client_set_str(char **ptr, const char *str, size_t len) {
    free(*ptr);
    if(str) {
        char *newvalue = malloc(len+1);
        if(!newvalue) {
            *ptr = NULL;
            return 1;
        }
        memcpy(newvalue, str, len);
        newvalue[len] = 0;
        *ptr = newvalue;
    } else {
        *ptr = NULL;
    }
    return 0;
}

int http_client_set_method_len(HttpClient *client, const char *method, size_t len) {
    return client_set_str(&client->method, method, len);
}

int http_client_set_uri_len(HttpClient *client, const char *uri, size_t len) {
    return client_set_str(&client->uri, uri, len);
}

int http_client_add_request_header(HttpClient *client, cxmutstr name, cxmutstr value) {
    return header_array_add(client->request_headers, name, value);
}

int http_client_add_request_header_copy(HttpClient *client, cxstring name, cxstring value) {
    if(!client->mp) {
        client->mp = cxMempoolCreate(64, CX_MEMPOOL_TYPE_PURE);
        if(!client->mp) {
            return 1;
        }
    }
    
    cxmutstr n = cx_strdup_a(client->mp->allocator, name);
    cxmutstr v = cx_strdup_a(client->mp->allocator, value);
    
    int err = 1;
    if(n.ptr && v.ptr) {
        err = http_client_add_request_header(client, n, v);
    }
    if(err) {
        cxFree(client->mp->allocator, n.ptr);
        cxFree(client->mp->allocator, v.ptr);
    }
    return err;
}

int http_client_set_content_length(HttpClient *client, int64_t contentlength) {
    client->req_content_length = contentlength;
    char ctlen_buf[32];
    size_t len = snprintf(ctlen_buf, 32, "%" PRId64, contentlength);
    return http_client_add_request_header_copy(client, cx_str("content-length"), cx_strn(ctlen_buf, len));
}

int http_client_enable_chunked_transfer_encoding(HttpClient *client) {
    client->req_content_length = -1;
    return http_client_add_request_header(client, cx_mutstr("transfer-encoding"), cx_mutstr("chunked"));
}

static int client_start_poll(HttpClient *client) {
    client->event.fn = client_connected;
    client->event.finish = client_finished;
    return ev_poll(client->ev, client->socketfd, &client->event);
}

int http_client_start(HttpClient *client) {
    client->event.events = EVENT_POLLOUT;
    client->event.cookie = client;
    if(client->socketfd != -1) {
        int ret = client_connected(client->ev, &client->event);
        if(ret != 0) {
            return client_start_poll(client);
        }
        return 0;
    }
    
    int socketfd = socket(client->domain, SOCK_STREAM, 0);
    if(socketfd < 0) {
        return 1;
    }
    if(util_socket_setnonblock(socketfd, 1)) {
        return 1;
    }
    client->socketfd = socketfd;
      
    int ret = 1;
    if(connect(socketfd, client->addr, client->addrlen)) {
        int err = errno;
        if(err == EINPROGRESS) {
            ret = client_start_poll(client);
        } else {
            log_ereport(LOG_FAILURE, "http-client-start: connect failed: %s", strerror(err));
        }
    } else {
        ret = 0; // TODO
    }
    
    if(ret) {
        close(socketfd);
        client->socketfd = -1;
    }
    return ret;
}

int http_client_process(HttpClient *client, Event *event) {
    int ret = client_process(client, event);
    if(ret && client->error == 0 && client->event.fn == NULL) {
        if(client_start_poll(client)) {
            client->error = 1;
        }
    }
    return ret;
}

size_t http_client_message_buf_size_available(HttpClient *client) {
    return client->transfer_buffer_alloc - client->transfer_buffer_len;
}

int http_client_add_message(HttpClient *client, const void *buf, size_t size) {
    size_t available = http_client_message_buf_size_available(client);
    if(available == 0) {
        return HTTP_CLIENT_CALLBACK_WOULD_BLOCK;
    }
    if(size > available) {
        size = available;
    }
    memcpy(client->transfer_buffer + client->transfer_buffer_len, buf, size);
    return size;
}

static int create_req_buffer(HttpClient *client) {
    CxBuffer buf;
    if(cxBufferInit(&buf, cxDefaultAllocator, NULL, HTTP_CLIENT_BUFFER_SIZE, CX_BUFFER_AUTO_EXTEND)) {
        return 1;
    }
    
    if(client->method) {
        cxBufferPutString(&buf, "GET ");
    } else {
        cxBufferPutString(&buf, client->method);
    }
    cxBufferPutString(&buf, client->uri ? client->uri : "/");
    cxBufferPutString(&buf, " HTTP/1.1\r\n");
    
    HeaderArray *hdr = client->request_headers;
    while(hdr) {
        for(int i=0;i<hdr->len;i++) {
            cxBufferPutString(&buf, hdr->headers[i].name);
            cxBufferPutString(&buf, ": ");
            cxBufferPutString(&buf, hdr->headers[i].value);
            cxBufferPutString(&buf, "\r\n");
        }
        hdr = hdr->next;
    }
    cxBufferPutString(&buf, "\r\n");
    client->transfer_buffer = buf.space;
    client->transfer_buffer_alloc = buf.capacity;
    client->transfer_buffer_len = buf.size;
    
    return 0;
}

static int client_connected(EventHandler *ev, Event *event) {
    HttpClient *client = event->cookie;
    client->last_event = event;
    if(client->stage < 0) {
        return 0;
    }
    if(create_req_buffer(client)) {
        // TODO: set error
        return 0; // end
    }
    event->fn = client_io;
    
    return client_io(ev, event);
}

static int client_io(EventHandler *ev, Event *event) {
    HttpClient *client = event->cookie;
    return client_process(client, event);
}

static int client_process(HttpClient *client, Event *event) {
    client->last_event = event;
    if(client->stage < 0) {
        return 0;
    }
    
    if(client->stage == 0) {
        if(client->transfer_buffer_pos < client->transfer_buffer_len) {
            if(client_send_buf(client)) {
                return client->error == 0;
            }
        }

        // do we need to send a request body?
        if(client->req_content_length != 0) {
            if(client_send_request_body(client)) {
                return client->error == 0;
            }
        }
    }
    
    // writing complete, switch to read events
    
    if(client->event.events != EVENT_POLLIN) {
        if(&client->event != event) {
            // The current event, that invoked client_process, is not the
            // actual HttpClient event.
            // Remove the current HttpClient poll event and re-add it later
            // if needed
            if(ev_remove_poll(client->ev, client->socketfd)) {
                client->error = 1;
                return 1;
            }
            client->event.fn = NULL;
        }
        client->event.events = EVENT_POLLIN;
        
    }
    
    client->stage = 1;
    
    if(client_read_response_header(client)) {  
        return client->error == 0;
    }
    int ret = 0;
    if(client->stage == 2) {
        // websocket: write message buffer
        ret = client_send_buf(client);
    }
    if(client_read_response_body(client)) {
        ret = 1;
    }
    
    if(ret) {
        return client->error == 0;
    }
    
    return 0;
}

static int client_finished(EventHandler *ev, Event *event) {
    HttpClient *client = event->cookie;
    
    close(client->socketfd);
    client->socketfd = -1;
    client->stage = -1;
    
    // request finished
    if(client->response_finished) {
        client->response_finished(client, client->response_finished_userdata);
    }
    
    return 0;
}

// sends the content of the transfer buffer to client->socketfd
static int client_send_buf(HttpClient *client) {
    size_t nbytes = client->transfer_buffer_len - client->transfer_buffer_pos;
    ssize_t w;
    while((w = write(client->socketfd, client->transfer_buffer + client->transfer_buffer_pos, nbytes)) > 0) {
        client->transfer_buffer_pos += w;
        nbytes = client->transfer_buffer_len - client->transfer_buffer_pos;
        if(nbytes == 0) {
            break;
        }
    }
    
    if(w <= 0) {
        if(errno != EAGAIN) {
            // TODO: log correct host
            log_ereport(LOG_FAILURE, "http-client %s - %s: write failed: %s", "localhost", client->uri, strerror(errno));
            client->error = 1;
        }
        return 1;
    }
    
    return client->transfer_buffer_pos < client->transfer_buffer_len;
}

static int client_send_request_body(HttpClient *client) {
    size_t rbody_readsize = client->transfer_buffer_alloc;
    size_t rbody_buf_offset = 0;
    if(client->req_content_length == -1) {
        // chunked transfer encoding:
        // don't fill req_buffer completely, reserve some space for
        // a chunk header, that will be inserted at the beginning
        rbody_readsize -= 16;
        rbody_buf_offset = 16;
    }
    while(!client->request_body_complete) {
        ssize_t r = client->request_body_read(client, client->transfer_buffer + rbody_buf_offset, rbody_readsize, client->request_body_read_userdata);
        if(r <= 0) {
            if(r == HTTP_CLIENT_CALLBACK_WOULD_BLOCK) {
                return 1;
            } else if(r == 0) {
                // EOF
                client->request_body_complete = 1;
                break;
            } else {
                // error
                client->error = 1;
                return 1;
            }
        } else if(client->req_content_length == -1 && r + 32 < rbody_readsize) {
            // is it time to terminate the request body?
            // try read some additional bytes, if it returns 0, we know
            // the request body is complete and we can add the termination chunk
            char *r2buf = client->transfer_buffer + rbody_buf_offset + r;
            ssize_t r2 = client->request_body_read(client, r2buf, 32, client->request_body_read_userdata);
            if(r > 0) {
                r += r2;
            } else if(r == 0) {
                memcpy(r2buf, "0\r\n\r\n", 5);
                r += 5;
                client->request_body_complete = 1;
                client->request_body_terminated = 1;
            } else if(r == HTTP_CLIENT_CALLBACK_WOULD_BLOCK) {
                return 1;
            } else {
                client->error = 1;
                return 1;
            }
        }

        size_t startpos = 0;
        if(client->req_content_length == -1) {
            char chunkheader[16];
            int chunkheaderlen = snprintf(chunkheader, 16, "%zx\r\n", (size_t)r);
            startpos = 16 - chunkheaderlen;
            memcpy(client->transfer_buffer + startpos, chunkheader, chunkheaderlen);
        }

        client->req_contentlength_pos += r;
        client->transfer_buffer_pos = startpos;
        client->transfer_buffer_len = rbody_buf_offset + r;
        if(client_send_buf(client)) {
            return 1;
        }
    }
    
    // chunked transfer encoding: terminate
    if(client->req_content_length == -1 && !client->request_body_terminated) {
        memcpy(client->transfer_buffer, "0\r\n\r\n", 5);
        client->transfer_buffer_pos = 0;
        client->transfer_buffer_len = 5;
        client->request_body_terminated = 1;
        if(client_send_buf(client)) {
            return 1;
        }
        
    } else if(client->req_content_length != client->req_contentlength_pos) {
        // incomplete request body
        client->error = 1;
        return 1;
    }
    
    return 0;
}

// returns 0 success
//         1 would block or error
static int client_read_response_header(HttpClient *client) {
    if(client->response_header_complete) {
        return 0;
    }
    
    unsigned char *buffer = client->buffer.inbuf + client->buffer.cursize;
    size_t nbytes = client->buffer.maxsize - client->buffer.cursize;
    
    ssize_t r;
    while((r = read(client->socketfd, client->buffer.inbuf + client->buffer.cursize, client->buffer.maxsize - client->buffer.cursize)) > 0) {     
        client->buffer.cursize += r;
        if(!client->response_header_complete) {
            switch(http_parser_process(client->parser)) {
                case 0: { // finish
                    if(!http_parser_validate(client->parser)) {
                        client->error = 1;
                        return 1;
                    }
                    client->statuscode = client->parser->status;
                    
                    client->response_header_complete = 1;
                    if(client->response_start) {
                        cxmutstr msg = client->parser->msg;
                        char t = msg.ptr[msg.length];
                        msg.ptr[msg.length] = 0;
                        int ret = client->response_start(client, client->statuscode, msg.ptr, client->response_start_userdata);
                        msg.ptr[msg.length] = t;
                        
                        if(ret != 0) {
                            if(ret != HTTP_CLIENT_CALLBACK_WOULD_BLOCK) {
                                client->error = 1;
                            }
                            return 1;
                        }
                    }
                    break;
                }
                case 1: { // need more data
                    continue;
                }
                case 2: { // error
                    client->error = 1;
                    return 1;
                }
            }
        }
        
        // header complete
        break;
    }
    
    if(r <= 0) {
        if(r == 0) {
            // unexpected EOF
            client->error = 1;
        } else if(errno != EAGAIN) {
            log_ereport(LOG_FAILURE, "http-client: IO error: %s", strerror(errno));
            client->error = 1;
        }
        return 1;
    }
    
    // initialize httpstream
    HeaderArray *headers = client->parser->headers;
    long long contentlength = 0;
    int chunkedtransferenc = 0;
    cxmutstr hdr_connection = CX_NULLSTR;
    cxmutstr hdr_upgrade = CX_NULLSTR;
    while(headers) {
        for(int i=0;i<headers->len;i++) {
            cxmutstr header = headers->headers[i].name;
            cxmutstr hvalue = headers->headers[i].value;
            if(!cx_strcasecmp(header, "content-length")) {
                if(!cx_strtoll(hvalue, &contentlength, 10)) {
                    headers = NULL;
                    break;
                }
            } else if(!cx_strcasecmp(header, "transfer-encoding")) {
                if(!cx_strcmp(hvalue, "chunked")) {
                    chunkedtransferenc = 1;
                    headers = NULL;
                    break;
                }
            } else if(!cx_strcasecmp(header, "connection")) {
                hdr_connection = hvalue;
            }
            
            if(client->statuscode == 101 && !cx_strcasecmp(header, "upgrade")) {
                hdr_upgrade = hvalue;
            }
        }
        
        if(headers) {
            headers = headers->next;
        }
    }
    
    if(client->statuscode == 101) {
        if(!cx_strcasecmp(hdr_upgrade, "websocket") && !cx_strcasecmp(hdr_connection, "upgrade")) {
            client->stage = 2;
            client->event.events = EVENT_POLLIN|EVENT_POLLOUT;
            
            // prepare IO buffers for websockets
            client->transfer_buffer_len = 0;
            client->transfer_buffer_pos = 0;
            
            client->transfer2_buffer = malloc(HTTP_CLIENT_BUFFER_SIZE);
            if(!client->transfer2_buffer) {
                client->error = 1;
                return 1;
            }
            client->transfer2_buffer_alloc = HTTP_CLIENT_BUFFER_SIZE;
            client->transfer2_buffer_len = 0;
            client->transfer2_buffer_pos = 0;
        } else {
            // error: unknown protocol
            log_ereport(LOG_FAILURE, "http-client: unknown protocol upgrade: %.*s", (int)hdr_upgrade.length, hdr_upgrade.ptr);
            client->error = 1;
            return 1;
        }
    } else if(contentlength > 0 || chunkedtransferenc) {
        IOStream *fd = Sysstream_new(NULL, client->socketfd);
        if(!fd) {
            client->error = 1;
            return 1;
        }
        HttpStream *http = (HttpStream*)httpstream_new(NULL, fd);
        if(!http) {
            fd->free(fd);
        }
        
        // we can reuse the already allocated transfer_bufer for transfer2
        client->transfer2_buffer = client->transfer_buffer;
        client->transfer2_buffer_alloc = client->transfer_buffer_alloc;
        client->transfer2_buffer_len = 0;
        client->transfer_buffer_pos = 0;
        client->transfer_buffer = NULL;
        client->transfer_buffer_alloc = 0;
        
        if(contentlength > 0) {
            http->max_read = contentlength;
            httpstream_enable_buffered_read(&http->st, (char*)client->buffer.inbuf, client->buffer.maxsize, &client->buffer.cursize, &client->buffer.pos);
        } else if(chunkedtransferenc) {
            httpstream_enable_chunked_read(&http->st, (char*)client->buffer.inbuf, client->buffer.maxsize, &client->buffer.cursize, &client->buffer.pos);
        }
        client->stream = http;
    }
    
    return 0;
}

// uses the response_body_write callback to write the content of the
// transfer buffer
// returns 0 success
//         1 would block or error
static int client_write_response(HttpClient *client) {
    while(client->transfer2_buffer_pos < client->transfer2_buffer_len) {
        char *buf = client->transfer2_buffer + client->transfer2_buffer_pos;
        size_t len = client->transfer2_buffer_len - client->transfer2_buffer_pos;
        int ret = client->response_body_write(client, buf, len, client->response_body_write_userdata);
        if(ret > 0) {
           client->transfer2_buffer_pos += ret; 
        } else if(ret == 0) {
            // EOF?
            // check if the write is incomplete, which would be an error
            client->error == client->transfer2_buffer_pos < client->transfer2_buffer_len;
            return client->error;
        } else {
            if(ret != HTTP_CLIENT_CALLBACK_WOULD_BLOCK) {
                client->error = 1;
            }
            return 1;
        }
    }
    return 0;
}

// returns 0 success
//         1 would block or error
static int client_read_response_body(HttpClient *client) {
    if(!client->stream) {
        return 0; // no input stream -> no response body
    }
    
    // does the transfer buffer still contains bytes, that should be written?
    if(client_write_response(client)) {
        return 1;
    }
    
    char *buf = client->transfer2_buffer;
    size_t nbytes = client->transfer2_buffer_alloc;
    
    ssize_t r;
    while((r = net_read(&client->stream->st, buf, nbytes)) > 0) {
        client->transfer2_buffer_len = r;
        client->transfer2_buffer_pos = 0;
        if(client->response_body_write) {
            if(client_write_response(client)) {
                return 1;
            }
        }
    }
    
    if(r < 0) {
        if(client->stream->st.io_errno != EWOULDBLOCK) {
            client->error = 1;
        }
        return 1;
    }
    
    return 0;
}

/* --------------------------------- Tests --------------------------------- */

static CX_TEST(test_http_client_send_request) {
    CX_TEST_DO {
        EventHandler dummy;
        HttpClient *client = http_client_new(&dummy);
        
        int fds[2];
        util_socketpair(fds);
        util_socket_setnonblock(fds[0], 1);
        util_socket_setnonblock(fds[1], 1);
        client->socketfd = fds[0];
        int sock = fds[1];
        
        // create a large test buffer, that is bigger than the socket buffer
        // 32mb should be enough
        size_t len = 32*1024*1024;
        char *str = malloc(len);
        // init the buffer with random data
        for(size_t i=0;i<len;i+=sizeof(int)) {
            int *p = (int*)(str+i);
            *p = rand();
        }
        
        client->transfer_buffer = str;
        client->transfer_buffer_len = len;
        
        // test client_send_request
        
        int ret = client_send_buf(client);
        // It is very likely that the first client_send_request call doesn't
        // fully write the request buffer to the socket
        // In that case it returns 1 but without the error flag
        CX_TEST_ASSERT(ret == 1 && !client->error);
        CX_TEST_ASSERT(client->transfer_buffer_pos > 0);
        CX_TEST_ASSERT(client->transfer_buffer_pos < len);
        
        // read the request buffer from sock and continue with client_send_request
        CxBuffer buf;
        cxBufferInit(&buf, cxDefaultAllocator, NULL, len, CX_BUFFER_AUTO_EXTEND|CX_BUFFER_FREE_CONTENTS);
        char tmpbuf[1024];
        int writes = 1;
        while(client->transfer_buffer_pos < client->transfer_buffer_len && writes < 2000000) {
            ssize_t r = read(sock, tmpbuf, 1024);
            CX_TEST_ASSERT(r >= 0);
            cxBufferWrite(tmpbuf, 1, r, &buf);
            ret = client_send_buf(client);
            CX_TEST_ASSERT(ret == 0 || (ret == 1 && !client->error));
            
            writes++;
        }
        CX_TEST_ASSERT(client->transfer_buffer_pos == client->transfer_buffer_len);
        
        // finish reading the request buffer from sock
        ssize_t r;
        while((r = read(sock, tmpbuf, 1024)) > 0 && writes < 2000000) {
            cxBufferWrite(tmpbuf, 1, r, &buf);
            writes++;
        }
        
        CX_TEST_ASSERT(buf.size == len);
        CX_TEST_ASSERT(!memcmp(str, buf.space, len));
        
        // cleanup
        close(fds[0]);
        close(fds[1]);
        http_client_free(client);
        cxBufferDestroy(&buf);
    }
}

typedef struct TestResponse {
    int status;
    char *msg;
    CxBuffer *response;
    int error_interval;
    int error_test;
} TestResponse;

static int test_response_start(HttpClient *client, int status, char *msg, void *userdata) {
    TestResponse *test = userdata;
    test->status = status;
    test->msg = strdup(msg);
    return 0;
}

static ssize_t test_response_body_write(HttpClient *client, void *buf, size_t size, void *userdata) {
    TestResponse *test = userdata;
    if(test->error_interval > 0 && test->error_test >= test->error_interval) {
        test->error_test = 0;
        return HTTP_CLIENT_CALLBACK_WOULD_BLOCK;
    }
    test->error_test++;
    
    cxBufferWrite(buf, 1, size, test->response);
    return size;
}


typedef struct TestRequestBody {
    char   *content;
    size_t length;
    size_t pos;
    int    chunksize;
    int    max_reads; // max number of reads until test_request_body_read returns 0
    int    cur_reads; // current number of read-attempts
} TestRequestBody;

static ssize_t test_request_body_read(HttpClient *client, void *buf, size_t size, void *userdata) {
    TestRequestBody *req = userdata;
    req->cur_reads++;
    if(req->chunksize == 0 || req->cur_reads > req->max_reads) {
        return -1;
    }
    size_t max = req->length - req->pos;
    if(max == 0) {
        return 0;
    }
    
    size_t sz = req->chunksize > size ? size : req->chunksize;
    if(sz > max) {
        sz = max;
    }
    memcpy(buf, req->content + req->pos, sz);
    req->pos += sz;
    return sz;
}

static CX_TEST(test_http_client_send_request_body_chunked) {
    CX_TEST_DO {
        EventHandler dummy;
        HttpClient *client = http_client_new(&dummy);
        create_req_buffer(client);
        client->req_content_length = -1;
        
        int fds[2];
        util_socketpair(fds);
        util_socket_setnonblock(fds[0], 1);
        util_socket_setnonblock(fds[1], 1);
        client->socketfd = fds[0];
        int sock = fds[1];
        
        // response buffer
        CxBuffer buf;
        cxBufferInit(&buf, cxDefaultAllocator, NULL, 1024, CX_BUFFER_AUTO_EXTEND|CX_BUFFER_FREE_CONTENTS);
        
        // test
        char request_body[1024];
        memset(request_body, 'x', 1024);
        memset(request_body+128, 'y', 128);
        memset(request_body+384, 'z', 128);
        memset(request_body+640, ':', 128);
        memset(request_body+896, '!', 128);
        
        TestRequestBody req;
        req.content = request_body;
        req.length = 1024;
        req.pos = 0;
        req.chunksize = 16;
        req.max_reads = 8;
        req.cur_reads = 0;
        client->request_body_read = test_request_body_read;
        client->request_body_read_userdata = &req;
        
        memset(client->transfer_buffer, '_', client->transfer_buffer_alloc);
        client->transfer_buffer_pos = 0;
        client->transfer_buffer_len = 0;
        
        // send the first 128 bytes
        while(req.cur_reads <= req.max_reads) {
            int ret = client_send_request_body(client);
            CX_TEST_ASSERT(ret == 1);
            CX_TEST_ASSERT(!client->error);
            char buf2[1024];
            ssize_t r = read(sock, buf2, 1024);
            if(r > 0) {
                cxBufferWrite(buf2, 1, r, &buf);
            }
        }
        
        // because we are using chunked transfer encoding, the result buffer
        // (buf) should contain more than 128 bytes (additional chunk headers)
        CX_TEST_ASSERT(buf.pos > 128);
        
        // change chunk size to 128
        req.max_reads = 9999;
        req.chunksize = 128;
        while(req.cur_reads <= req.max_reads) {
            int ret = client_send_request_body(client);
            CX_TEST_ASSERT(!client->error);
            char buf2[2048];
            ssize_t r;
            while((r = read(sock, buf2, 2048)) > 0) {
                cxBufferWrite(buf2, 1, r, &buf);
            }
            if(ret == 0) {
                break;
            }
        }
        CX_TEST_ASSERT(req.cur_reads < req.max_reads);
        
        // verify chunks
        char test_request_body[1024];
        memset(test_request_body, 0, 1024);
        
        int pos = 0;
        int chunklen = 0;
        char *str = buf.space;
        while(str < buf.space + buf.size) {
            cxstring chunkheader = cx_strn(str, 2);
            if(!cx_strcmp(chunkheader, "0\r")) {
                chunkheader.length = 1;
            }
            int ret = cx_strtoi(chunkheader, &chunklen, 16);
            CX_TEST_ASSERT(ret == 0);
            if(chunklen == 0) {
                break;
            }
            
            char *data = str + 4;
            CX_TEST_ASSERT(data + chunklen < buf.space + buf.size);
            memcpy(test_request_body + pos, data, chunklen);
            pos += chunklen;
            str = data + chunklen;
        }
        CX_TEST_ASSERT(!memcmp(request_body, test_request_body, 1024));
        
        // cleanup
        close(fds[0]);
        close(fds[1]);
        http_client_free(client);
        cxBufferDestroy(&buf);
    }
}

static CX_TEST_SUBROUTINE(test_read_response, cxstring response_str, CxBuffer *response_body) {
    EventHandler dummy;
    HttpClient *client = http_client_new(&dummy);
    create_req_buffer(client);
    client->transfer_buffer_pos = 0;
    client->transfer_buffer_len = 0;
    client->req_content_length = -1;

    int fds[2];
    util_socketpair(fds);
    util_socket_setnonblock(fds[0], 1);
    util_socket_setnonblock(fds[1], 1);
    client->socketfd = fds[0];
    int sock = fds[1];
    
    TestResponse testr = { 0 };
    testr.response = response_body;
    client->response_body_write = test_response_body_write;
    client->response_body_write_userdata = &testr;

    // test

    size_t response_pos = 0;
    while(response_pos < response_str.length) {
        size_t nbytes = response_str.length - response_pos;
        ssize_t w = write(sock, response_str.ptr + response_pos, nbytes);
        if(w > 0) {
            response_pos += w;
        }

        if(!client->response_header_complete) {
            int ret = client_read_response_header(client);
            CX_TEST_ASSERT(client->error == 0);
            if(ret == 1) {
                continue;
            }
        }
        
        if(response_body != NULL) {
            CX_TEST_ASSERT(client->stream != NULL);
            
            int ret = client_read_response_body(client);
            CX_TEST_ASSERT(client->error == 0);
            if(ret == 1) {
                continue;
            }
        } else {
            CX_TEST_ASSERT(client->stream == NULL);
        }

        break;
    }

    // cleanup
    close(fds[0]);
    close(fds[1]);
    http_client_free(client);
}

static CX_TEST(test_http_client_read_response_head) {
    CX_TEST_DO {
        char *response_str = 
                "HTTP/1.1 204 OK\r\n"
                "Host: localhost\r\n"
                "Content-length: 0\r\n"
                "\r\n";
        
        CX_TEST_CALL_SUBROUTINE(test_read_response, cx_str(response_str), NULL);
        
        response_str = 
                "HTTP/1.1 204 OK\r\n"
                "Host: localhost\r\n"
                "\r\n";
        
        CX_TEST_CALL_SUBROUTINE(test_read_response, cx_str(response_str), NULL);
    }
}

static CX_TEST(test_http_client_read_response_ctlen) {
    CX_TEST_DO {
        char *response_str = 
                "HTTP/1.1 200 OK\r\n"
                "Host: localhost\r\n"
                "Content-length: 13\r\n"
                "\r\n"
                "Hello World!\n";
        CxBuffer *buf = cxBufferCreate(NULL, NULL, 1024, CX_BUFFER_AUTO_EXTEND|CX_BUFFER_FREE_CONTENTS);
        
        CX_TEST_CALL_SUBROUTINE(test_read_response, cx_str(response_str), buf);
        CX_TEST_ASSERT(buf->size == 13);
        CX_TEST_ASSERT(!cx_strcmp(cx_strn(buf->space, buf->size), "Hello World!\n"));
        
        cxBufferFree(buf);
    }
}

static CX_TEST(test_http_client_read_response_ctlen_big) {
    CX_TEST_DO {
        // create response body
        size_t len = 1024*1024*32;
        char *response_str = malloc(len);
        char *str = response_str;
        for(size_t i=0;i<len;i+=sizeof(int)) {
            int *p = (int*)(str+i);
            *p = rand();
        }
        cxstring body = cx_strn(response_str, len);
        
        // create request string
        CxBuffer *req = cxBufferCreate(NULL, NULL, 1024, CX_BUFFER_AUTO_EXTEND|CX_BUFFER_FREE_CONTENTS);
        cxBufferPutString(req,
                "HTTP/1.1 200 OK\r\n"
                "Host: localhost\r\n"
                "Content-length: ");
        char ctlen[32];
        snprintf(ctlen, 32, "%d\r\n\r\n", (int)len);
        cxBufferPutString(req, ctlen);
        cxBufferPutString(req, body);
        cxBufferTerminate(req);
        
        // response buffer
        CxBuffer *buf = cxBufferCreate(NULL, NULL, 1024, CX_BUFFER_AUTO_EXTEND|CX_BUFFER_FREE_CONTENTS);
        
        // test
        CX_TEST_CALL_SUBROUTINE(test_read_response, cx_strn(req->space, req->size), buf);
        CX_TEST_ASSERT(buf->size == len);
        CX_TEST_ASSERT(!cx_strcmp(cx_strn(buf->space, buf->size), body));
        
        cxBufferFree(req);
        cxBufferFree(buf);
    }
}

static CX_TEST_SUBROUTINE(test_http_client_io, cxstring response, int status_code, const char *msg, CxBuffer *out_buf, size_t write_blocksz, int error_interval) {
    EventHandler dummy;
    HttpClient *client = http_client_new(&dummy);

    int fds[2];
    util_socketpair(fds);
    util_socket_setnonblock(fds[0], 1);
    util_socket_setnonblock(fds[1], 1);
    client->socketfd = fds[0];
    int sock = fds[1];

    // setup client
    http_client_set_uri(client, "/test/uri/");
    http_client_set_method(client, "GET");
    http_client_add_request_header(client, cx_mutstr("Host"), cx_mutstr("localhost"));
    http_client_add_request_header(client, cx_mutstr("Test1"), cx_mutstr("value1"));
    http_client_add_request_header(client, cx_mutstr("Test2"), cx_mutstr("value2"));
    create_req_buffer(client);

    size_t req_header_len = client->transfer_buffer_len;

    TestResponse testr = { 0 };
    testr.response = out_buf;
    testr.error_interval = error_interval;
    client->response_start = test_response_start;
    client->response_start_userdata = &testr;
    client->response_body_write = test_response_body_write;
    client->response_body_write_userdata = &testr;

    // test IO
    Event event;
    event.cookie = client;
    int ret = client_io(&dummy, &event);
    CX_TEST_ASSERT(!client->error);
    CX_TEST_ASSERT(ret == 1);

    // do IO and read request until the header is processed
    size_t req_header_pos = 0;
    char req_buf[4];
    while(req_header_pos < req_header_len) {
        ssize_t r = read(sock, req_buf, 4);
        if(r == 0) {
            break;
        }
        CX_TEST_ASSERT(r > 0);
        req_header_pos += r;
        ret = client_io(&dummy, &event);
        CX_TEST_ASSERT(!client->error);
        CX_TEST_ASSERT(ret == 1);
    }
    CX_TEST_ASSERT(req_header_pos == req_header_len);
    
    const char *response_str = response.ptr;
    size_t response_str_len = response.length;
    size_t response_str_pos = 0;

    // send response and do IO
    int in_progress = 1;
    while(in_progress) {
        size_t len = response_str_len - response_str_pos;
        if(len > 0) {
            if(len > write_blocksz) {
                len = write_blocksz;
            }
            ssize_t w = write(sock, response_str + response_str_pos, len);
            if(w == 0) {
                break;
            }
            CX_TEST_ASSERT(w > 0);
            response_str_pos += w;
        }
        
        ret = client_io(&dummy, &event);
        if(ret == 0) {
            in_progress = 0;
        }

        CX_TEST_ASSERT(!client->error);
    }
    CX_TEST_ASSERT(response_str_pos == response_str_len);
    CX_TEST_ASSERT(testr.status == status_code);
    CX_TEST_ASSERT(testr.msg);
    CX_TEST_ASSERT(!strcmp(testr.msg, msg));

    // cleanup
    free(testr.msg);
    close(fds[0]);
    close(fds[1]);
    http_client_free(client);
}

static CX_TEST_SUBROUTINE(test_http_client_io_simple, size_t blocksz, int error_interval) {
    cxstring response_str = cx_str(
            "HTTP/1.1 200 OK\r\n"
            "Host: localhost\r\n"
            "Content-length: 13\r\n"
            "\r\n"
            "Hello World!\n");
    CxBuffer *buf = cxBufferCreate(NULL, NULL, 1024, CX_BUFFER_AUTO_EXTEND|CX_BUFFER_FREE_CONTENTS);
    
    CX_TEST_CALL_SUBROUTINE(test_http_client_io, response_str, 200, "OK", buf, blocksz, error_interval);
    CX_TEST_ASSERT(buf->size == 13);
    CX_TEST_ASSERT(!cx_strcmp(cx_strn(buf->space, buf->size), "Hello World!\n"));
    
    cxBufferFree(buf);
}

static CX_TEST(test_http_client_io_simple_1b) {
    CX_TEST_DO {
        CX_TEST_CALL_SUBROUTINE(test_http_client_io_simple, 1, 0);
    }
}

static CX_TEST(test_http_client_io_simple_2b) {
    CX_TEST_DO {
        CX_TEST_CALL_SUBROUTINE(test_http_client_io_simple, 2, 0);
    }
}

static CX_TEST(test_http_client_io_simple_3b) {
    CX_TEST_DO {
        CX_TEST_CALL_SUBROUTINE(test_http_client_io_simple, 3, 0);
    }
}

static CX_TEST(test_http_client_io_simple_16b) {
    CX_TEST_DO {
        CX_TEST_CALL_SUBROUTINE(test_http_client_io_simple, 16, 0);
    }
}

static CX_TEST(test_http_client_io_simple_512b) {
    CX_TEST_DO {
        CX_TEST_CALL_SUBROUTINE(test_http_client_io_simple, 512, 0);
    }
}

static CX_TEST_SUBROUTINE(test_http_client_io_large, size_t blocksz) {
    int ctlen = 1024*1024*32;
    CxBuffer *resp = cxBufferCreate(NULL, NULL, ctlen + 128, CX_BUFFER_AUTO_EXTEND|CX_BUFFER_FREE_CONTENTS);
    cx_bprintf(resp,
            "HTTP/1.1 200 OK\r\n"
            "Host: localhost\r\n"
            "Content-length: %d\r\n"
            "\r\n",
            ctlen);
    size_t content_start = resp->pos;
    CX_TEST_ASSERT(resp->pos + ctlen < resp->capacity);
    memset(resp->space + resp->pos, 'x', ctlen);
    resp->pos += ctlen;
    resp->size += ctlen;
    
    CxBuffer *buf = cxBufferCreate(NULL, NULL, 1024, CX_BUFFER_AUTO_EXTEND|CX_BUFFER_FREE_CONTENTS);
    
    CX_TEST_CALL_SUBROUTINE(test_http_client_io, cx_strn(resp->space, resp->size), 200, "OK", buf, blocksz, 0);
    CX_TEST_ASSERT(buf->size == ctlen);
    CX_TEST_ASSERT(!memcmp(buf->space, resp->space + content_start, ctlen));
    
    cxBufferFree(resp);
    cxBufferFree(buf);
}

static CX_TEST(test_http_client_io_large_128b) {
    CX_TEST_DO {
        CX_TEST_CALL_SUBROUTINE(test_http_client_io_large, 128);
    }
}

static CX_TEST(test_http_client_io_large_4096b) {
    CX_TEST_DO {
        CX_TEST_CALL_SUBROUTINE(test_http_client_io_large, 1024);
    }
}

static CX_TEST_SUBROUTINE(test_http_client_io_chunked_transfer, size_t blocksz, int error_interval) {
    cxstring response_str = cx_str( 
            "HTTP/1.1 200 OK\r\n"
            "Host: localhost\r\n"
            "Transfer-encoding: chunked\r\n"
            "\r\n"
            "d\r\n"        
            "Hello World!\n"
            "\r\n"
            "0\r\n\r\n");
    CxBuffer *buf = cxBufferCreate(NULL, NULL, 1024, CX_BUFFER_AUTO_EXTEND|CX_BUFFER_FREE_CONTENTS);
    
    CX_TEST_CALL_SUBROUTINE(test_http_client_io, response_str, 200, "OK", buf, blocksz, error_interval);
    CX_TEST_ASSERT(buf->size == 13);
    CX_TEST_ASSERT(!cx_strcmp(cx_strn(buf->space, buf->size), "Hello World!\n"));
    
    cxBufferFree(buf);
}

static CX_TEST(test_http_client_io_chunked_transfer_1b) {
    CX_TEST_DO {
        CX_TEST_CALL_SUBROUTINE(test_http_client_io_chunked_transfer, 1, 0);
    }
}

static CX_TEST(test_http_client_io_chunked_transfer_2b) {
    CX_TEST_DO {
        CX_TEST_CALL_SUBROUTINE(test_http_client_io_chunked_transfer, 2, 0);
    }
}

static CX_TEST(test_http_client_io_chunked_transfer_8b) {
    CX_TEST_DO {
        CX_TEST_CALL_SUBROUTINE(test_http_client_io_chunked_transfer, 16, 0);
    }
}

static CX_TEST(test_http_client_io_chunked_transfer_64b) {
    CX_TEST_DO {
        CX_TEST_CALL_SUBROUTINE(test_http_client_io_chunked_transfer, 64, 0);
    }
}

static CX_TEST_SUBROUTINE(test_http_client_io_large_chunked_transfer, size_t blocksz, int error_interval) {
    int chunk1 = 1024*1024*2;
    int chunk2 = 1024*128;
    int chunk3 = 123;
    int chunk4 = 1024*1024*5;
    int chunk5 = 1024*16;
    
    char *chunkbuf = malloc(chunk4);
    
    int ctlen = chunk1 + chunk2 + chunk3 + chunk4 + chunk5;
    CxBuffer *resp = cxBufferCreate(NULL, NULL, ctlen * 128, CX_BUFFER_AUTO_EXTEND|CX_BUFFER_FREE_CONTENTS);
    cx_bprintf(resp,
            "HTTP/1.1 200 OK\r\n"
            "Transfer-encoding: chunked\r\n"
            "\r\n",
            ctlen);
    size_t content_start = resp->pos;
    CX_TEST_ASSERT(resp->pos + ctlen < resp->capacity);
    
    memset(chunkbuf, 'x', chunk1);
    cx_bprintf(resp, "%x\r\n", chunk1);
    int chunk1_pos = resp->pos;
    cxBufferWrite(chunkbuf, 1, chunk1, resp);
    cxBufferPutString(resp, "\r\n");
    
    memset(chunkbuf, '2', chunk2);
    cx_bprintf(resp, "%x\r\n", chunk2);
    int chunk2_pos = resp->pos;
    cxBufferWrite(chunkbuf, 1, chunk2, resp);
    cxBufferPutString(resp, "\r\n");
    
    memset(chunkbuf, 'c', chunk3);
    cx_bprintf(resp, "%x\r\n", chunk3);
    int chunk3_pos = resp->pos;
    cxBufferWrite(chunkbuf, 1, chunk3, resp);
    cxBufferPutString(resp, "\r\n");
    
    memset(chunkbuf, '4', chunk4);
    cx_bprintf(resp, "%x\r\n", chunk4);
    int chunk4_pos = resp->pos;
    cxBufferWrite(chunkbuf, 1, chunk4, resp);
    cxBufferPutString(resp, "\r\n");
    
    memset(chunkbuf, 'z', chunk5);
    cx_bprintf(resp, "%x\r\n", chunk5);
    int chunk5_pos = resp->pos;
    cxBufferWrite(chunkbuf, 1, chunk5, resp);
    cxBufferPutString(resp, "\r\n0\r\n\r\n");
    
    
    CxBuffer *buf = cxBufferCreate(NULL, NULL, 1024, CX_BUFFER_AUTO_EXTEND|CX_BUFFER_FREE_CONTENTS);
    
    CX_TEST_CALL_SUBROUTINE(test_http_client_io, cx_strn(resp->space, resp->size), 200, "OK", buf, blocksz, error_interval);
    CX_TEST_ASSERT(buf->size == ctlen);
    CX_TEST_ASSERT(!memcmp(buf->space, resp->space + chunk1_pos, chunk1));
    CX_TEST_ASSERT(!memcmp(buf->space + chunk1, resp->space + chunk2_pos, chunk2));
    CX_TEST_ASSERT(!memcmp(buf->space + chunk1 + chunk2, resp->space + chunk3_pos, chunk3));
    CX_TEST_ASSERT(!memcmp(buf->space + chunk1 + chunk2 + chunk3, resp->space + chunk4_pos, chunk4));
    CX_TEST_ASSERT(!memcmp(buf->space + chunk1 + chunk2 + chunk3 + chunk4, resp->space + chunk5_pos, chunk5));
    
    cxBufferFree(resp);
    cxBufferFree(buf);
}

static CX_TEST(test_http_client_io_large_chunked_transfer_1024b) {
    CX_TEST_DO {
        CX_TEST_CALL_SUBROUTINE(test_http_client_io_large_chunked_transfer, 1024, 0);
    }
}

static CX_TEST(test_http_client_io_write_error1) {
    CX_TEST_DO {
        CX_TEST_CALL_SUBROUTINE(test_http_client_io_simple, 1, 1);
    }
}

static CX_TEST(test_http_client_io_write_error2) {
    CX_TEST_DO {
        CX_TEST_CALL_SUBROUTINE(test_http_client_io_simple, 1, 2);
    }
}

static CX_TEST(test_http_client_io_write_error3) {
    CX_TEST_DO {
        CX_TEST_CALL_SUBROUTINE(test_http_client_io_simple, 1, 3);
    }
}

static CX_TEST(test_http_client_io_write_blsz8_error1) {
    CX_TEST_DO {
        CX_TEST_CALL_SUBROUTINE(test_http_client_io_simple, 8, 1);
    }
}

static CX_TEST(test_http_client_io_write_blsz8_error2) {
    CX_TEST_DO {
        CX_TEST_CALL_SUBROUTINE(test_http_client_io_simple, 8, 2);
    }
}

static CX_TEST(test_http_client_io_write_blsz8_error3) {
    CX_TEST_DO {
        CX_TEST_CALL_SUBROUTINE(test_http_client_io_simple, 8, 3);
    }
}

void http_client_add_tests(CxTestSuite *suite) {
    cx_test_register(suite, test_http_client_send_request);
    cx_test_register(suite, test_http_client_send_request_body_chunked);
    cx_test_register(suite, test_http_client_read_response_head);
    cx_test_register(suite, test_http_client_read_response_ctlen);
    cx_test_register(suite, test_http_client_read_response_ctlen_big);
    cx_test_register(suite, test_http_client_io_simple_1b);
    cx_test_register(suite, test_http_client_io_simple_2b);
    cx_test_register(suite, test_http_client_io_simple_3b);
    cx_test_register(suite, test_http_client_io_simple_16b);
    cx_test_register(suite, test_http_client_io_simple_512b);
    cx_test_register(suite, test_http_client_io_large_128b);
    cx_test_register(suite, test_http_client_io_large_4096b);
    cx_test_register(suite, test_http_client_io_chunked_transfer_1b);
    cx_test_register(suite, test_http_client_io_chunked_transfer_2b);
    cx_test_register(suite, test_http_client_io_chunked_transfer_8b);
    cx_test_register(suite, test_http_client_io_chunked_transfer_64b);
    cx_test_register(suite, test_http_client_io_large_chunked_transfer_1024b);
    cx_test_register(suite, test_http_client_io_write_error1);
    cx_test_register(suite, test_http_client_io_write_error2);
    cx_test_register(suite, test_http_client_io_write_error3);
    cx_test_register(suite, test_http_client_io_write_blsz8_error1);
    cx_test_register(suite, test_http_client_io_write_blsz8_error2);
    cx_test_register(suite, test_http_client_io_write_blsz8_error3);
}