src/server/daemon/httplistener.c

/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 2013 Olaf Wintermann. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *   1. Redistributions of source code must retain the above copyright
 *      notice, this list of conditions and the following disclaimer.
 *
 *   2. Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in the
 *      documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include "../public/nsapi.h"

#include <stdio.h>
#include <stdlib.h>

#include <sys/shm.h>
#include <sys/ipc.h>
#include <sys/file.h>
#include <netinet/tcp.h>
#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#include <unistd.h>
#include <strings.h>
#include <stdbool.h>
#include <signal.h>
#include <pthread.h>

#include <stdio.h>
#include <stdlib.h>

#include <openssl/rand.h> /* RAND_bytes, connection id */

#include <cx/hash_map.h>

#include "../util/atomic.h"
#include "httplistener.h"
#include "netsite.h"

#include "session.h"
#include "configmanager.h"
#include "log.h"

#define LISTENER_MAX_PROTOCOL_TOKENS 1024


#define LISTENER_PROTO_IPV4 "ipv4"
#define LISTENER_PROTO_IPV6 "ipv6"


/*
 * key: string  format: <protocol>:<port>
 * value: WSSocket*
 * 
 * protocol: ipv4 | ipv6
 * port: short
 */
static CxMap *listener_socket_map;

static pthread_mutex_t listener_mutex = PTHREAD_MUTEX_INITIALIZER;

static void wssocket_free(WSSocket *s) {
    close(s->socket);
    free(s);
}

int http_listener_global_init(void) {
    listener_socket_map = cxHashMapCreate(cxDefaultAllocator, CX_STORE_POINTERS, 4);
    if(!listener_socket_map) {
        return 1;
    }
    listener_socket_map->simple_destructor = (cx_destructor_func)wssocket_free;
    
    return 0;
}

void http_listener_global_shutdown(void) {
    cxMapDestroy(listener_socket_map);
}

int start_all_listener() {
    ServerConfiguration *conf = cfgmgr_get_server_config();
    CxList *ls = conf->listeners;
    CxIterator iter = cxListIterator(ls);
    cx_foreach(HttpListener *, listener, iter) {
        http_listener_start(listener);
    }

    return 0;
}

static HttpSSL* create_http_ssl(ListenerConfig *conf) {
    SSL_CTX *ctx = SSL_CTX_new(SSLv23_server_method());
    if(!ctx) {
        return NULL;
    }
    SSL_CTX_set_options(
            ctx,
            SSL_OP_SINGLE_DH_USE | SSL_OP_NO_SSLv3);
    SSL_CTX_set_mode(ctx, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
    SSL_CTX_set_mode(ctx, SSL_MODE_ENABLE_PARTIAL_WRITE);
    
    int error = 0;
    if(conf->disable_proto.ptr) {
        cxstring *plist = NULL;
        ssize_t n = cx_strsplit_a(cxDefaultAllocator, conf->disable_proto, cx_str(","), LISTENER_MAX_PROTOCOL_TOKENS, &plist);
        if(plist) {
            for(int i=0;i<n;i++) {
                cxstring proto = plist[i];
                log_ereport(
                        LOG_VERBOSE,
                        "Listener %s: Disable protocol %s",
                        conf->name.ptr,
                        proto.ptr);
                if(!cx_strcasecmp(cx_strtrim(proto), cx_str("SSLv2"))) {
                    SSL_CTX_set_options(ctx, SSL_OP_NO_SSLv2);
                } else if(!cx_strcasecmp(cx_strtrim(proto), cx_str("SSLv3"))) {
                    SSL_CTX_set_options(ctx, SSL_OP_NO_SSLv3);
                } else if(!cx_strcasecmp(cx_strtrim(proto), cx_str("TLSv1"))) {
                    SSL_CTX_set_options(ctx, SSL_OP_NO_TLSv1);
                } else if(!cx_strcasecmp(cx_strtrim(proto), cx_str("TLSv1.1"))) {
#ifdef SSL_OP_NO_TLSv1_1
                    SSL_CTX_set_options(ctx, SSL_OP_NO_TLSv1_1);
#else
                    log_ereport(
                            LOG_WARN,
                            "Listener: %s: TLSv1.1 not supported",
                            conf->name.ptr);
#endif
                } else if(cx_strcasecmp(cx_strtrim(proto), cx_str("TLSv1.2"))) {
#ifdef SSL_OP_NO_TLSv1_2
                    SSL_CTX_set_options(ctx, SSL_OP_NO_TLSv1_2);
#else
                    log_ereport(
                            LOG_WARN,
                            "Listener: %s: TLSv1.2 not supported",
                            conf->name.ptr);
#endif
                } else if(cx_strcasecmp(cx_strtrim(proto), cx_str("TLSv1.3"))) {
#ifdef SSL_OP_NO_TLSv1_3
                    SSL_CTX_set_options(ctx, SSL_OP_NO_TLSv1_3);
#else
                    log_ereport(
                            LOG_WARN,
                            "Listener: %s: TLSv1.3 not supported",
                            conf->name.ptr);
#endif
                } else {
                    error = 1;
                    log_ereport(
                            LOG_MISCONFIG,
                            "Listener: %s: Unknown protocol %s",
                            conf->name.ptr,
                            proto.ptr);
                }
            }
            free(plist);
        }
    }
    
    if(error) {
        SSL_CTX_free(ctx);
        return NULL;
    }
    
    int ret;
    char errbuf[512];
    
    // get TLS cert
    error = 0;
    if(conf->chainfile.ptr) {
        ret = SSL_CTX_use_certificate_chain_file(ctx, conf->chainfile.ptr);
        if(!ret) { 
            ERR_error_string(ERR_get_error(), errbuf);
            log_ereport(LOG_MISCONFIG, "Cannot load ssl cert file: %s", errbuf);
            error = 1;
        }
    } else if(conf->certfile.ptr) {
        ret = SSL_CTX_use_certificate_file(ctx, conf->certfile.ptr, SSL_FILETYPE_PEM);
        if(!ret) {
            ERR_error_string(ERR_get_error(), errbuf);
            log_ereport(LOG_MISCONFIG, "Cannot load ssl chain file: %s", errbuf);
            error = 1;
        }
    } else {
        log_ereport(LOG_MISCONFIG, "Listener %s: no CertChain/Cert specified", conf->name.ptr);
        error = 1;
    }

    // get private key
    ret = SSL_CTX_use_PrivateKey_file(ctx, conf->privkeyfile.ptr, SSL_FILETYPE_PEM);
    if(!ret) { 
        ERR_error_string(ERR_get_error(), errbuf);
        log_ereport(LOG_MISCONFIG, "Cannot load ssl key file: %s", errbuf);
        error = 1;
    }
    
    if(error) {
        SSL_CTX_free(ctx);
        return NULL;
    }
    
    HttpSSL *ssl = pool_malloc(conf->cfg->pool, sizeof(HttpSSL));
    if(!ssl) {
        SSL_CTX_free(ctx);
        return NULL;
    }
    ZERO(ssl, sizeof(HttpSSL));
    ssl->sslctx = ctx;
    
    return ssl;
}

static WSSocket* create_socket(ListenerConfig *conf, const char *protocol) {
    WSBool ipv4 = !strcmp(protocol, "ipv4") ? TRUE : FALSE;
    
    int s = -1;
    if(ipv4) {
        // ipv4 socket
        s = socket(AF_INET, SOCK_STREAM, 0);
    } else {
        // ipv6 socket
        s = socket(AF_INET6, SOCK_STREAM, IPPROTO_TCP);
    }
    if(s < 0) {
        log_ereport(
                LOG_FAILURE,
                "cannot create socket: protocol: %s port: %d error: %s",
                protocol,
                conf->port,
                strerror(errno));
        return NULL;
    }
    
    // socket options
    int o = 1;
    setsockopt(
            s,
            SOL_SOCKET, SO_REUSEADDR,
            &o,
            sizeof(int));
    
#ifdef LINUX
    if(!ipv4) {
        o = 1;
        setsockopt(
                s,
                IPPROTO_IPV6,
                IPV6_V6ONLY,
                &o,
                sizeof(int));
    }
#endif
    
    // bind server socket to address
    union ws_socketaddr addr;
    struct sockaddr *servaddr;
    size_t servaddr_size;
    if(ipv4) {
        // ipv4
        memset(&addr.addr4, 0, sizeof(addr.addr4));
        addr.addr4.sin_family = AF_INET;
        addr.addr4.sin_addr.s_addr = htonl(INADDR_ANY);
        addr.addr4.sin_port = htons(conf->port);
        servaddr = (struct sockaddr *)&addr.addr4;
        servaddr_size = sizeof(addr.addr4);
    } else {
        // ipv6
        memset(&addr.addr6, 0, sizeof(addr.addr6));
        addr.addr6.sin6_family = AF_INET6;
        addr.addr6.sin6_addr = in6addr_any;
        addr.addr6.sin6_port = htons(conf->port);
        servaddr = (struct sockaddr *)&addr.addr6;
        servaddr_size = sizeof(addr.addr6);
    }
    
    if(bind(s, servaddr, servaddr_size)) {
        log_ereport(
                LOG_FAILURE,
                "cannot bind socket: protocol: %s port: %d error: %s",
                protocol,
                conf->port,
                strerror(errno));
        close(s);
        return NULL;
    }
    
    // everything is ok, create WSSocket object
    WSSocket *wssocket = malloc(sizeof(WSSocket));
    if(!wssocket) {
        close(s);
        return NULL;
    }
    ZERO(wssocket, sizeof(WSSocket));
    wssocket->socket = s;
    wssocket->addr = addr;
    if(ipv4) {
        wssocket->sockaddr = (struct sockaddr *)&wssocket->addr.addr4;
    } else {
        wssocket->sockaddr = (struct sockaddr *)&wssocket->addr.addr6;
    }
    wssocket->sockaddr_size = servaddr_size;
    
    return wssocket;
}

static WSSocket* get_socket(ListenerConfig *conf, const char *protocol) {
    char key_data[32];
    size_t key_len = snprintf(key_data, 32, "%s:%d", protocol, conf->port);
    CxHashKey key = cx_hash_key(key_data, key_len);
    
    WSSocket *sock = cxMapGet(listener_socket_map, key);
    if(!sock) {
        sock = create_socket(conf, protocol);
        if(sock) {
            cxMapPut(listener_socket_map, key, sock);
        }
    }
    return sock;
}

/*
 * returns HttpSSL of socket1 or socket2 if ssl is available
 */
static HttpSSL* socket_get_ssl(WSSocket *socket1, WSSocket *socket2) {
    if(socket1 && socket1->ssl) {
        return socket1->ssl;
    }
    if(socket2 && socket2->ssl) {
        return socket2->ssl;
    }
    return NULL;
}

static HttpListener* listener_create(ListenerConfig *conf) {
    pool_handle_t *pool = conf->cfg->pool;
    HttpListener *listener = pool_malloc(pool, sizeof(HttpListener));
    if(!listener) {
        return NULL;
    }
    ZERO(listener, sizeof(HttpListener));
    
    listener->running = 0;
    listener->cfg = conf->cfg;
    listener->name = cx_strdup_a(pool_allocator(pool), cx_strcast(conf->name));
    listener->default_vs.vs_name = pool_strdup(pool, conf->vs.ptr);
    listener->threadpool = NULL;
    if(conf->threadpool.ptr != NULL) {
        listener->threadpool = get_threadpool(cx_strcast(conf->threadpool));
    }
    if(listener->threadpool == NULL) {
        listener->threadpool = get_default_threadpool();
    }
    if(conf->blockingio) {
        listener->session_handler = create_basic_session_handler(pool);
    } else {
        listener->session_handler = create_event_session_handler(pool);
    }
    listener->nacceptors = conf->nacceptors;
    listener->port = conf->port;
    listener->next = NULL;
    //listener->ssl = NULL;
    
    // create sockets
    listener->server_socket = get_socket(conf, LISTENER_PROTO_IPV4);
    listener->server_socket6 = get_socket(conf, LISTENER_PROTO_IPV6);
    if(!listener->server_socket && !listener->server_socket6) {
        log_ereport(LOG_FAILURE, "Listener %s: no server socket", conf->name.ptr);
        return NULL;
    }
    
    if(listener->server_socket) {
        wssocket_ref(listener->server_socket);
    }
    if(listener->server_socket6) {
        wssocket_ref(listener->server_socket6);
    }
    
    // create SSL context
    if(conf->ssl) {
        HttpSSL *ssl = socket_get_ssl(listener->server_socket, listener->server_socket6);
        if(!ssl) {
            ssl = create_http_ssl(conf);
            if(!ssl) {
                log_ereport(LOG_FAILURE, "Listener %s: cannot create SSL context", conf->name.ptr);
                return NULL;
            }
        }
        if(listener->server_socket) {
            listener->server_socket->ssl = ssl;
        }
        if(listener->server_socket6) {
            listener->server_socket6->ssl = ssl;
        }
    }

    // create acceptors
    listener->acceptors = calloc(listener->nacceptors, sizeof(void*));
    listener->acceptors6 = calloc(listener->nacceptors, sizeof(void*));
    for (int i=0;i<listener->nacceptors;i++) {
        listener->acceptors[i] = acceptor_new(listener);
        listener->acceptors6[i] = acceptor_new(listener);
        listener->acceptors6[i]->ipv6 = TRUE;
    }

    return listener;
}

HttpListener* http_listener_create(ListenerConfig *conf) {
    pthread_mutex_lock(&listener_mutex);
    HttpListener *listener = listener_create(conf);
    pthread_mutex_unlock(&listener_mutex);
    return listener;
}

void http_listener_destroy(HttpListener *listener) {
    log_ereport(LOG_DEBUG, "destroy http listener: %s config: %p", listener->name.ptr, listener->cfg);
    
    if(listener->shutdown) {
        pthread_mutex_destroy(&listener->shutdown_mutex);
        pthread_cond_destroy(&listener->shutdown_cond);
        
        free(listener->acceptors);
        free(listener->acceptors6);
    }
}

int http_listener_start(HttpListener *listener) {
    if(listener->running) {
        return 0;
    }
    log_ereport(LOG_INFORM, "start listener on port %d", listener->port);
    
    WSBool ipv4 = listener->server_socket ? TRUE : FALSE;
    WSBool ipv6 = listener->server_socket6 ? TRUE: FALSE;

    if (ipv4 && !listener->server_socket->listening && listen(listener->server_socket->socket, 256) == -1) {
        log_ereport(LOG_FAILURE, "http_listener_start: listen failed: %s", strerror(errno));
        return -1;
    } else {
        listener->server_socket->listening = TRUE;
    }
    if (ipv6 && !listener->server_socket6->listening && listen(listener->server_socket6->socket, 256) == -1) {
        log_ereport(LOG_FAILURE, "http_listener_start: listen v6 failed: %s", strerror(errno));
        return -1;
    } else {
        listener->server_socket6->listening = TRUE;
    }

    // start acceptor threads
    for (int i=0;i<listener->nacceptors;i++) {
        if(ipv4) {
            acceptor_start(listener->acceptors[i]);
        }
        if(ipv6) {
            acceptor_start(listener->acceptors6[i]);
        }
    }
    listener->running = TRUE;

    return 0;
}

int http_listener_socket_eq(HttpListener *l1, HttpListener *l2) {
    if(l1->server_socket && l2->server_socket && l1->server_socket == l2->server_socket) {
        return TRUE;
    }
    if(l1->server_socket6 && l2->server_socket6 && l1->server_socket6 == l2->server_socket6) {
        return TRUE;
    }
    return FALSE;
}

void http_listener_set_next(HttpListener *listener, HttpListener *next) {
    while(listener->next) {
        listener = listener->next;
    }
    listener->next = next;
}

int http_listener_connect(HttpListener *listener, WSBool ipv6) {
    int domain = ipv6 ? AF_INET6 : AF_INET;
    int client = socket(domain, SOCK_STREAM, 0);
    if(client < 0) {
        return -1;
    }
    
    struct sockaddr *sockaddr;
    size_t sockaddr_size;
    if(ipv6) {
        sockaddr = listener->server_socket6->sockaddr;
        sockaddr_size = listener->server_socket6->sockaddr_size;
    } else {
        sockaddr = listener->server_socket->sockaddr;
        sockaddr_size = listener->server_socket->sockaddr_size;
    }
    
    if(connect(client, sockaddr, sockaddr_size) < 0) {
        close(client);
        return -1;
    }
    
    return client;
}


void http_listener_shutdown_acceptors(HttpListener *listener) {
    if(pthread_mutex_init(&listener->shutdown_mutex, NULL)) {
        log_ereport(LOG_FAILURE, "http_listener_shutdown_acceptors: pthread_mutex_init failed: %s", strerror(errno));
        return;
    }
    if(pthread_cond_init(&listener->shutdown_cond, NULL)) {
        log_ereport(LOG_FAILURE, "http_listener_shutdown_acceptors: pthread_cond_init failed: %s", strerror(errno));
        return;
    }
    listener->shutdown = TRUE;
    
    log_ereport(LOG_INFORM, "shutdown http listener %s", listener->name.ptr);
    
    pthread_mutex_lock(&listener->shutdown_mutex);
    
    // shut down acceptors by connecting to the server socket, to make sure
    // accept() returns
    // the acceptor will handle this as a normal connection, but this should
    // not be a problem
    for(int i=0;i<listener->nacceptors;i++) {
        listener->acceptors[i]->exit = TRUE;
        int client4 = http_listener_connect(listener, FALSE);
        if(client4 < 0) {
            log_ereport(LOG_FAILURE, "http_listener_shutdown_acceptors: cannot connect to ipv4 server socket: %s", strerror(errno));
        } else {
            close(client4);
        }
        
        listener->acceptors6[i]->exit = TRUE;
        int client6 = http_listener_connect(listener, TRUE);
        if(client6 < 0) {
            log_ereport(LOG_FAILURE, "http_listener_shutdown_acceptors: cannot connect to ipv6 server socket: %s", strerror(errno));
        } else {
            close(client6);
        }
    }
    
    // The last acceptor will notify listener_shutdown.cond
    // It is not really necessary to wait for acceptors to shut down,
    // we do it mostly for nicer log messages. The timeout is there
    // for making sure, this function never blocks forever.
    struct timespec ts;
    clock_gettime(CLOCK_REALTIME, &ts);
    ts.tv_sec += 60;
    pthread_cond_timedwait(&listener->shutdown_cond, &listener->shutdown_mutex, &ts);
    

    pthread_mutex_unlock(&listener->shutdown_mutex);
}


Acceptor* acceptor_new(HttpListener *listener) {
    Acceptor *acceptor = malloc(sizeof(Acceptor));
    acceptor->listener = listener;
    acceptor->ipv6 = WS_FALSE;
    acceptor->exit = WS_FALSE;
    return acceptor;
}

void acceptor_start(Acceptor *a) {
    cfg_ref(a->listener->cfg);
    if(pthread_create(
            &a->tid,
            NULL,
            (void*(*)(void*))acceptor_thread,
            a) != 0)
    {
        log_ereport(LOG_FAILURE, "Listener %s: acceptor_start: %s acceptor", a->listener->name.ptr, strerror(errno));
        cfg_unref(a->listener->cfg);
    } else {
        (void)pthread_detach(a->tid);
    }
}

void* acceptor_thread(Acceptor *acceptor) {
    WS_ASSERT(acceptor);
    WS_ASSERT(acceptor->listener);
    WS_ASSERT(acceptor->listener->cfg);
    WS_ASSERT(acceptor->listener->session_handler);
    WS_ASSERT(acceptor->listener->session_handler->enqueue_connection);
    
    acceptor->running = TRUE;
    HttpListener *listener = acceptor->listener;
    
    int server_socket;
    uint32_t *acceptors_running = &listener->nacceptors_running;
    
    ConnectionAddr ca;
    struct sockaddr *ca_ptr;
    socklen_t ca_length;
    ConnectionAddrType addr_type;
    HttpSSL *socket_ssl = NULL;
    if(acceptor->ipv6) {
        server_socket = listener->server_socket6->socket;
        ca_ptr = (struct sockaddr*)&ca.address_v6;
        ca_length = sizeof(ca.address_v6);
        addr_type = CONN_ADDR_IPV6;
        socket_ssl = listener->server_socket6->ssl;
    } else {
        server_socket = listener->server_socket->socket;
        ca_ptr = (struct sockaddr*)&ca.address_v4;
        ca_length = sizeof(ca.address_v4);
        addr_type = CONN_ADDR_IPV4;
        socket_ssl = listener->server_socket->ssl;
    }
    
    log_ereport(LOG_DEBUG, "acceptor: %p listener: %p start", acceptor, acceptor->listener);
    ws_atomic_inc32(acceptors_running);
    
    // in case the system is too busy and we get EMFILE (Too many open files)
    // we wait for busy_sleep_time seconds
    // the sleep time is increased after 5 attempts
    int busy_sleep_time = 5;
    int sleep_counter = 0;
    
    for (;;) {
        // accept connections
        int clientfd;
        socklen_t length = ca_length;

        // accept a connection
        clientfd = accept(
                server_socket,
                ca_ptr,
                &length);
        log_ereport(LOG_DEBUG, "acceptor: %p listener: %p: accept(): %d", acceptor, acceptor->listener, clientfd);
        if (clientfd == -1) {
            log_ereport(LOG_FAILURE, "accept %s failed: %s", acceptor->ipv6 ? "ipv6" : "ipv4", strerror(errno));
            
            switch(errno) {
                case EBADF: acceptor->exit = 1; break;
                case EINTR: continue;
                case EINVAL: acceptor->exit = 1; break;
                case EMFILE: {
                    sleep(busy_sleep_time);
                    sleep_counter++;
                    if(sleep_counter >= 5) {
                        sleep_counter = 0;
                        busy_sleep_time += 5;
                    }
                }
            }
            
            if(acceptor->exit) {
                log_ereport(LOG_VERBOSE, "acceptor thread %p: listener: %p exit", acceptor, acceptor->listener);
                break;
            }
            
            
            continue;
        }
        
        //if(http_listener_apply_keep_alive_settings(listener, clientfd)) {
        //    close(clientfd);
        //    continue;
        //}
             
        // check listener
        HttpListener *ls = listener;
        int acceptor_exit = 0;
        while(ls->next) {
            ls = ls->next;
            acceptor_exit = 1;
        }
        
        // create Connection object
        Connection *conn = malloc(sizeof(Connection));
        RAND_bytes((unsigned char*)&conn->id, sizeof(conn->id));
        conn->id <<= 16;
        conn->address = ca;
        conn->addr_type = addr_type;
        conn->fd = clientfd;
        conn->listener = ls;
        conn->ssl_accepted = 0;
        conn->ssl_error = 0;
        log_ereport(LOG_DEBUG, "trace reqid: %016llx connection accepted", (unsigned long long int)conn->id);
        if(socket_ssl) {
            // SSL connections are always non-blocking 
            // set socket non blocking
            int flags;
            if((flags = fcntl(conn->fd, F_GETFL, 0)) == -1) {
                flags = 0;
            }
            if(fcntl(conn->fd, F_SETFL, flags | O_NONBLOCK)) {
                log_ereport(LOG_DEBUG, "trace reqid: %016llx connection destroyed | fcntl error", (unsigned long long int)conn->id);
                log_ereport(LOG_FAILURE, "acceptor: fcntl failed: %s", strerror(errno));
                close(clientfd);
                free(conn);
                conn = NULL;
            } else {
                SSL *ssl = SSL_new(socket_ssl->sslctx);
                if(ssl) {
                    SSL_set_fd(ssl, clientfd);

                    conn->ssl = ssl;
                    conn->read = connection_ssl_read;
                    conn->write = connection_ssl_write;
                    conn->close = connection_ssl_close;
                } else {
                    log_ereport(LOG_DEBUG, "trace reqid: %016llx connection destroyed | SSL init error", (unsigned long long int)conn->id);
                    log_ereport(LOG_FAILURE, "acceptor: %p listener: %p SSL_new() failed", acceptor, acceptor->listener);
                    free(conn);
                    close(clientfd);
                    conn = NULL;
                }
            }
        } else {
            conn->ssl = NULL;
            conn->read = connection_read;
            conn->write = connection_write;
            conn->close = connection_close;
        }
        
        if(conn) {
            cfg_ref(ls->cfg);

            // enqueue the connection
            ls->session_handler->enqueue_connection(
                    ls->session_handler,
                    conn);
        }

        // ready for new connection
        
        if(acceptor_exit || acceptor->exit) {
            // this acceptor is outdated
            log_ereport(LOG_VERBOSE, "acceptor thread %p: listener: %p exit", acceptor, acceptor->listener);
            break;
        }
        
        busy_sleep_time = 5; // reset sleep time
        sleep_counter = 0;
    }
    
    if(ws_atomic_dec32(acceptors_running) == 0) {
        // notify
        if(listener->shutdown) {
            log_ereport(LOG_DEBUG, "last acceptor shutdown: notify cfgmgr");
            pthread_mutex_lock(&listener->shutdown_mutex);
            pthread_cond_signal(&listener->shutdown_cond);
            pthread_mutex_unlock(&listener->shutdown_mutex);
        }
    }
    
    acceptor->running = FALSE;
    
    cfg_unref(acceptor->listener->cfg);
    
    free(acceptor);
    
    return NULL;
}

void wssocket_ref(WSSocket *ws) {
    ws_atomic_inc32(&ws->ref);
}

void wssocket_unref(WSSocket *ws) {
    // does nothing yet, because maybe it is not a good idea to destroy
    // a socket
    ws_atomic_dec32(&ws->ref);
}


int http_listener_apply_keep_alive_settings(HttpListener *listener, int fd) {
    // TODO: all these values should be configurable
    int optval = 1;
    if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &optval, sizeof(optval))) {
        log_ereport(LOG_FAILURE, "listener: cannot set SO_KEEPALIVE: %s", strerror(errno));
        return 1;
    }

    int keepidle = 240;
    if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &keepidle, sizeof(keepidle))) {
        log_ereport(LOG_FAILURE, "listener: cannot set TCP_KEEPIDLE to value %d: %s", keepidle, strerror(errno));
        return 1;
    }

    int keepintvl = 10;
    if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &keepintvl, sizeof(keepintvl))) {
        log_ereport(LOG_FAILURE, "listener: cannot set TCP_KEEPINTVL to value %d: %s", keepintvl, strerror(errno));
        return 1;
    }

    int keepcnt = 3;
    if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &keepcnt, sizeof(keepcnt))) {
        log_ereport(LOG_FAILURE, "listener: cannot set TCP_KEEPCNT to value %d: %s", keepcnt, strerror(errno));
        return 1;
    }
    
    return 0;
}