# HG changeset patch
# User Olaf Wintermann <olaf.wintermann@gmail.com>
# Date 1670086835 -3600
# Node ID a28a5ccc894b05f2534f9ebfdb28414246d31c41
# Parent  02b003f7560cb23a0b916de4cd0fc86f8147fe27
improve http listener migration / shutdown of old listener

diff -r 02b003f7560c -r a28a5ccc894b src/server/daemon/config.c
--- a/src/server/daemon/config.c	Sat Dec 03 16:31:08 2022 +0100
+++ b/src/server/daemon/config.c	Sat Dec 03 18:00:35 2022 +0100
@@ -392,6 +392,12 @@
             cfg_unref(cfg->next);
         }
         log_ereport(LOG_VERBOSE, "destroy configuration %p", cfg);
+        
+        CxIterator i = cxListIterator(cfg->listeners, 0);
+        cx_foreach(HttpListener*, listener, i) {
+            http_listener_destroy(listener);
+        }
+        
         pool_destroy(cfg->pool);
     }
 }
diff -r 02b003f7560c -r a28a5ccc894b src/server/daemon/httplistener.c
--- a/src/server/daemon/httplistener.c	Sat Dec 03 16:31:08 2022 +0100
+++ b/src/server/daemon/httplistener.c	Sat Dec 03 18:00:35 2022 +0100
@@ -420,6 +420,15 @@
     return listener;
 }
 
+void http_listener_destroy(HttpListener *listener) {
+    log_ereport(LOG_DEBUG, "destroy http listener: %s config: %p", listener->name.ptr, listener->cfg);
+    
+    if(listener->shutdown) {
+        pthread_mutex_destroy(&listener->shutdown_mutex);
+        pthread_cond_destroy(&listener->shutdown_cond);
+    }
+}
+
 int http_listener_start(HttpListener *listener) {
     if(listener->running) {
         return 0;
@@ -500,8 +509,24 @@
 
 
 void http_listener_shutdown_acceptors(HttpListener *listener) {
-    // 
+    if(pthread_mutex_init(&listener->shutdown_mutex, NULL)) {
+        log_ereport(LOG_FAILURE, "http_listener_shutdown_acceptors: pthread_mutex_init failed: %s", strerror(errno));
+        return;
+    }
+    if(pthread_cond_init(&listener->shutdown_cond, NULL)) {
+        log_ereport(LOG_FAILURE, "http_listener_shutdown_acceptors: pthread_cond_init failed: %s", strerror(errno));
+        return;
+    }
+    listener->shutdown = TRUE;
     
+    log_ereport(LOG_INFORM, "shutdown http listener %s", listener->name.ptr);
+    
+    pthread_mutex_lock(&listener->shutdown_mutex);
+    
+    // shut down acceptors by connecting to the server socket, to make sure
+    // accept() returns
+    // the acceptor will handle this as a normal connection, but this should
+    // not be a problem
     for(int i=0;i<listener->nacceptors;i++) {
         listener->acceptors[i]->exit = TRUE;
         int client4 = http_listener_connect(listener, FALSE);
@@ -519,6 +544,16 @@
             close(client6);
         }
     }
+    
+    // The last acceptor will notify listener_shutdown.cond
+    // It is not really necessary to wait for acceptors to shut down,
+    // we do it mostly for nicer log messages. The timeout is there
+    // for making sure, this function never blocks forever.
+    struct timespec ts;
+    ts.tv_sec = time(NULL) + 60;
+    pthread_cond_timedwait(&listener->shutdown_cond, &listener->shutdown_mutex, &ts);
+    
+    pthread_mutex_unlock(&listener->shutdown_mutex);
 }
 
 
@@ -540,6 +575,8 @@
     {
         log_ereport(LOG_FAILURE, "Listener %s: acceptor_start: %s acceptor", a->listener->name.ptr, strerror(errno));
         cfg_unref(a->listener->cfg);
+    } else {
+        (void)pthread_detach(a->tid);
     }
 }
 
@@ -554,6 +591,7 @@
     HttpListener *listener = acceptor->listener;
     
     int server_socket;
+    uint32_t *acceptors_running = &listener->nacceptors_running;
     
     ConnectionAddr ca;
     struct sockaddr *ca_ptr;
@@ -572,6 +610,7 @@
     }
     
     log_ereport(LOG_DEBUG, "acceptor: %p listener: %p start", acceptor, acceptor->listener);
+    ws_atomic_inc32(acceptors_running);
     
     for (;;) {
         // accept connections
@@ -662,6 +701,16 @@
         }
     }
     
+    if(ws_atomic_dec32(acceptors_running) == 0) {
+        // notify
+        if(listener->shutdown) {
+            log_ereport(LOG_DEBUG, "last acceptor shutdown: notify cfgmgr");
+            pthread_mutex_lock(&listener->shutdown_mutex);
+            pthread_cond_signal(&listener->shutdown_cond);
+            pthread_mutex_unlock(&listener->shutdown_mutex);
+        }
+    }
+    
     acceptor->running = FALSE;
     
     cfg_unref(acceptor->listener->cfg);
diff -r 02b003f7560c -r a28a5ccc894b src/server/daemon/httplistener.h
--- a/src/server/daemon/httplistener.h	Sat Dec 03 16:31:08 2022 +0100
+++ b/src/server/daemon/httplistener.h	Sat Dec 03 18:00:35 2022 +0100
@@ -57,8 +57,6 @@
 typedef struct _ws_socket        WSSocket;
 
 
-
-
 union vs {
     VirtualServer    *vs;
     char             *vs_name;
@@ -100,8 +98,13 @@
     Acceptor             **acceptors;
     Acceptor             **acceptors6;
     int                  nacceptors;
+    uint32_t             nacceptors_running;
     int                  running;
     HttpSSL              *ssl;
+    
+    pthread_mutex_t      shutdown_mutex;
+    pthread_cond_t       shutdown_cond;
+    WSBool               shutdown;
 };
 
 struct _http_ssl {
@@ -144,6 +147,8 @@
 
 HttpListener* http_listener_create(ListenerConfig *conf);
 
+void http_listener_destroy(HttpListener *listener);
+
 int http_listener_start(HttpListener *listener);
 
 
diff -r 02b003f7560c -r a28a5ccc894b src/server/daemon/httprequest.c
--- a/src/server/daemon/httprequest.c	Sat Dec 03 16:31:08 2022 +0100
+++ b/src/server/daemon/httprequest.c	Sat Dec 03 18:00:35 2022 +0100
@@ -420,12 +420,15 @@
     if(transfer_encoding) {
         if(!strcmp(transfer_encoding, "chunked")) {
             netbuf *nb = sn->netbuf;
+            // a separate buffer is required for reading chunked transfer enc
             sn->buffer = pool_malloc(pool, nb->maxsize);
             if(!sn->buffer) {
                 pool_destroy(pool);
                 // TODO: error 500 
                 return 1;
             }
+            
+            // copy remaining bytes from inbuf to the additional buffer
             if(nb->cursize - nb->pos > 0) {
                 memcpy(sn->buffer, nb->inbuf, nb->cursize);
             }
@@ -433,6 +436,7 @@
             sn->pos = nb->pos;
             sn->cursize = nb->cursize;
             
+            // clear inbuf
             nb->pos = 0;
             nb->cursize = 0;