Mercurial > hg > webserver / changeset

--- a/src/server/daemon/acl.c	Wed May 22 15:05:06 2013 +0200
+++ b/src/server/daemon/acl.c	Sun May 26 12:12:07 2013 +0200
@@ -105,14 +105,10 @@
         char *usr;
         char *pw;
         if(!basicauth_getuser(sn, rq, &usr, &pw)) {
-            user = list->defaultauthdb->get_user(list->defaultauthdb, usr);
+            int pwok;
+            user = authdb_get_and_verify(list->defaultauthdb, usr, pw, &pwok);
             if(!user) {
-                // wrong user name
-                return NULL;
-            }
-            if(!user->verify_password(user, pw)) {
-                // wrong password
-                user->free(user);
+                // wrong user or wrong password
                 return NULL;
             }
             // ok - user is authenticated

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/server/daemon/auth.c	Sun May 26 12:12:07 2013 +0200
@@ -0,0 +1,329 @@
+/*
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ *
+ * Copyright 2013 Olaf Wintermann. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   1. Redistributions of source code must retain the above copyright
+ *      notice, this list of conditions and the following disclaimer.
+ *
+ *   2. Redistributions in binary form must reproduce the above copyright
+ *      notice, this list of conditions and the following disclaimer in the
+ *      documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <pthread.h>
+
+#include "../ucx/map.h"
+#include "../util/atomic.h"
+#include "auth.h"
+
+static pthread_mutex_t auth_cache_mutex = PTHREAD_MUTEX_INITIALIZER;
+static UserCache cache;
+
+void auth_cache_init() {
+    printf("auth_cache_init\n");
+    // TODO: config parameters
+    //pthread_mutex_init(&auth_cache_mutex, NULL);
+    cache.map = calloc(64, sizeof(UserCacheElm));
+    cache.size = 96;
+    cache.count = 0;
+    cache.max_users = 64;
+    cache.head = NULL;
+    cache.trail = NULL;
+}
+
+User* auth_cache_get(char *authdb, char *user) {
+    //printf("auth_cache_get: %s\n", user);
+    /*
+     * create the key to access the map
+     * key: authdb\0user
+     */
+    size_t authdblen = strlen(authdb);
+    size_t userlen = strlen(user);
+    
+    size_t keylen = authdblen + userlen + 1;
+    char *key = malloc(keylen);
+    memcpy(key, authdb, authdblen);
+    key[authdblen] = 0;
+    memcpy(key + authdblen + 1, user, userlen);
+    
+    UcxKey mapkey = ucx_key(key, keylen);
+    
+    // get cached user from map
+    time_t now = time(NULL);
+    size_t slot = mapkey.hash%cache.size;
+    
+    pthread_mutex_lock(&auth_cache_mutex);
+    
+    UserCacheElm *elm = cache.map[slot];
+    while(elm && elm->key.hash != mapkey.hash) {
+        elm = elm->next_elm;
+    }
+    // if we have an elm, the hash is correct
+    if(elm) {
+        // compare the key data to be sure it is the correct user
+        int n = (mapkey.len > elm->key.len) ? elm->key.len : mapkey.len;
+        if (memcmp(elm->key.data, mapkey.data, n)) {
+            free(key);
+            pthread_mutex_unlock(&auth_cache_mutex);
+            return NULL;
+        }
+    } else {
+        free(key);
+        pthread_mutex_unlock(&auth_cache_mutex);
+        return NULL;
+    }
+    
+    // elm is now the correct UserCacheElm
+    // TODO: use configuration for expire time
+    if(now - elm->created > 120) {
+        // cached user expired
+        // remove all users from the list from the first to this one
+        UserCacheElm *e = cache.head;
+        while(e) {
+            if(e == elm) {
+                break;
+            }
+            UserCacheElm *n = e->next_user;
+            auth_cache_remove_from_map(e);
+            e = n;
+        }
+        cache.head = elm->next_user;
+        if(cache.trail == elm) {
+            cache.trail = NULL;
+        }
+        auth_cache_remove_from_map(elm);
+        free(key);
+        pthread_mutex_unlock(&auth_cache_mutex);
+        return NULL;
+    }
+    
+    pthread_mutex_unlock(&auth_cache_mutex);
+    free(key);
+    return (User*)elm->user;
+}
+
+void auth_cache_add(
+        char *authdb,
+        User *user,
+        char *password,
+        char **groups,
+        size_t numgroups)
+{
+    //printf("auth_cache_add: %s\n", user->name);
+    /*
+     * this function does not check, if the user is already in the map
+     * use it only after auth_cache_get
+     */
+    
+    CachedUser *cusr = malloc(sizeof(CachedUser));
+    cusr->user.name = strdup(user->name);
+    cusr->user.uid = user->uid;
+    cusr->user.gid = user->gid;
+    cusr->user.verify_password = 
+            (user_verify_passwd_f)cached_user_verify_password;
+    cusr->user.check_group = (user_check_group_f)cached_user_check_group;
+    cusr->user.free = (user_free_f)cached_user_unref;
+    
+    cusr->authdb = strdup(authdb);
+    cusr->password = strdup(password);
+    cusr->groups = calloc(numgroups, sizeof(sstr_t));
+    cusr->numgroups = numgroups;
+    for(int i=0;i<numgroups;i++) {
+        cusr->groups[i] = sstrdup(sstr(groups[i]));
+    }
+    cusr->ref = 1;
+    
+    /*
+     * add the user to the auth cache
+     * the auth cache is a list of all cached users
+     */
+    
+    // create list element
+    time_t now = time(NULL);
+    UserCacheElm *elm = malloc(sizeof(UserCacheElm));
+    elm->user = cusr;
+    elm->created = now;
+    elm->next_elm = NULL;
+    elm->next_user = NULL;
+    
+    // create map key
+    size_t authdblen = strlen(authdb);
+    size_t userlen = strlen(user->name);
+    size_t keylen = authdblen + userlen + 1;
+    char *key = malloc(keylen);
+    memcpy(key, authdb, authdblen);
+    key[authdblen] = 0;
+    memcpy(key + authdblen + 1, user->name, userlen);
+    UcxKey mapkey = ucx_key(key, keylen);
+    
+    elm->key = mapkey;
+    elm->slot = mapkey.hash%cache.size;
+    
+    // add user to list and map
+    pthread_mutex_lock(&auth_cache_mutex);
+    
+    // remove the first cached user if expired or the cache is full 
+    if(cache.head && 
+            (cache.count >= cache.max_users || now-cache.head->created > 120))
+    {
+        UserCacheElm *first = cache.head;
+        cache.head = first->next_user;
+        if(!cache.head) {
+            cache.trail = NULL;
+        }
+        auth_cache_remove_from_map(first);
+    }
+    
+    // add to map
+    UserCacheElm *prevelm = cache.map[elm->slot];
+    if(prevelm) {
+        for(;;) {
+            if(!prevelm->next_elm) {
+                break;
+            }
+            prevelm = prevelm->next_elm;
+        }
+    }
+    if(prevelm) {
+        prevelm->next_elm = elm;
+    } else {
+        cache.map[elm->slot] = elm;
+    }
+    
+    // add to list
+    if(cache.head) {
+        cache.trail->next_user = elm;
+        cache.trail = elm;
+    } else {
+        cache.head = elm;
+        cache.trail = elm;
+    }
+    
+    cache.count++;
+    
+    pthread_mutex_unlock(&auth_cache_mutex);
+}
+
+void auth_cache_remove_from_map(UserCacheElm *elm) {
+    UserCacheElm *prevelm = NULL;
+    UserCacheElm *e = cache.map[elm->slot];
+    while(e) {
+        if(e == elm) {
+            break;
+        } else {
+            prevelm = e;
+        }
+        e = e->next_elm;
+    }
+    if(prevelm) {
+        prevelm->next_elm = elm->next_elm;
+    } else {
+        cache.map[elm->slot] = elm->next_elm;
+    }
+    
+    free(elm->key.data);
+    cached_user_unref(elm->user);
+    free(elm);
+    
+    cache.count--;
+}
+
+int cached_user_verify_password(CachedUser *user, char *password) {
+    if(!strcmp(user->password, password)) {
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
+int cached_user_check_group(CachedUser *user, char *group) {
+    sstr_t grp = sstr(group);
+    for(int i=0;i<user->numgroups;i++) {
+        if(!sstrcmp(user->groups[i], grp)) {
+            return 1;
+        }
+    }
+    return 0;
+}
+
+void cached_user_unref(CachedUser *user) {
+    uint32_t ref = ws_atomic_dec32(&user->ref);
+    if(ref == 0) {
+        cached_user_delete(user);
+    }
+}
+
+void cached_user_delete(CachedUser *user) {
+    free(user->user.name);
+    free(user->authdb);
+    free(user->password);
+    free(user->groups);
+    free(user);
+}
+
+
+/*
+ * public API
+ * from public/auth.h
+ */
+
+User* authdb_get_user(AuthDB *db, char *user) {
+    if(db->use_cache) {
+        User *u = auth_cache_get(db->name, user);
+        if(u) {
+            return u;
+        }
+    }
+    return db->get_user(db, user);
+}
+
+User* authdb_get_and_verify(AuthDB *db, char *user, char *password, int *pw) {
+    User *u = NULL;
+    // try getting the user from the cache
+    if(db->use_cache) {
+        u = auth_cache_get(db->name, user);
+        if(u) {
+            if(u->verify_password(u, password)) {
+                *pw = 1;
+            } else {
+                *pw = 0;
+                u->free(u);
+                u = NULL;
+            }
+            return u;
+        }
+    }
+    // user not cached
+    u = db->get_user(db, user);
+    if(u) {
+        if(u->verify_password(u, password)) {
+            if(db->use_cache) {
+                auth_cache_add(db->name, u, password, NULL, 0);
+            }
+            *pw = 1;
+        } else {
+            *pw = 0;
+            u->free(u);
+            u = NULL;
+        }
+    }
+    return u;
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/server/daemon/auth.h	Sun May 26 12:12:07 2013 +0200
@@ -0,0 +1,89 @@
+/*
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ *
+ * Copyright 2013 Olaf Wintermann. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   1. Redistributions of source code must retain the above copyright
+ *      notice, this list of conditions and the following disclaimer.
+ *
+ *   2. Redistributions in binary form must reproduce the above copyright
+ *      notice, this list of conditions and the following disclaimer in the
+ *      documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef AUTH_H
+#define	AUTH_H
+
+#include <sys/types.h>
+#include "../public/auth.h"
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+    User     user;
+    char     *authdb;
+    sstr_t   *groups;
+    size_t   numgroups;
+    char     *password;
+    uint32_t ref;
+} CachedUser;
+
+typedef struct user_cache_elm UserCacheElm;
+struct user_cache_elm {
+    CachedUser   *user;
+    UserCacheElm *next_user; // next elm in the cached user list
+    UcxKey       key;        // key to access this element
+    size_t       slot;       // slot in the map
+    UserCacheElm *next_elm;  // next element in this map slot
+    time_t       created;
+};
+
+typedef struct {
+    UserCacheElm  **map;
+    size_t        size;
+    size_t        count;
+    size_t        max_users;
+    UserCacheElm  *head;
+    UserCacheElm  *trail;
+} UserCache;
+
+void auth_cache_init();
+
+User* auth_cache_get(char *authdb, char *user);
+void auth_cache_add(
+        char *authdb,
+        User *user,
+        char *password,
+        char **groups,
+        size_t numgroups);
+
+void auth_cache_remove_from_map(UserCacheElm *elm);
+
+int cached_user_verify_password(CachedUser *user, char *password);
+int cached_user_check_group(CachedUser *user, char *group);
+void cached_user_unref(CachedUser *user);
+void cached_user_delete(CachedUser *user);
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* AUTH_H */
+

--- a/src/server/daemon/authdb.c	Wed May 22 15:05:06 2013 +0200
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,34 +0,0 @@
-/*
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
- *
- * Copyright 2013 Olaf Wintermann. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   1. Redistributions of source code must retain the above copyright
- *      notice, this list of conditions and the following disclaimer.
- *
- *   2. Redistributions in binary form must reproduce the above copyright
- *      notice, this list of conditions and the following disclaimer in the
- *      documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "../ucx/map.h"
-#include "../public/auth.h"
-

--- a/src/server/daemon/keyfile_auth.c	Wed May 22 15:05:06 2013 +0200
+++ b/src/server/daemon/keyfile_auth.c	Sun May 26 12:12:07 2013 +0200
@@ -39,6 +39,7 @@
 Keyfile* keyfile_new() {
     Keyfile *keyfile = malloc(sizeof(Keyfile));
     keyfile->authdb.get_user = keyfile_get_user;
+    keyfile->authdb.use_cache = 0;
     keyfile->users = ucx_map_new(16);
     keyfile->ref = 1;
     return keyfile;

--- a/src/server/daemon/keyfile_auth.h	Wed May 22 15:05:06 2013 +0200
+++ b/src/server/daemon/keyfile_auth.h	Sun May 26 12:12:07 2013 +0200
@@ -52,7 +52,7 @@
 struct keyfile_user {
     User                 user;
     sstr_t               *groups;
-    size_t               numgroups; 
+    size_t               numgroups;
     enum KeyfileHashType hash_type;
     char                 *hash;
     size_t               hashlen;

--- a/src/server/daemon/ldap_auth.c	Wed May 22 15:05:06 2013 +0200
+++ b/src/server/daemon/ldap_auth.c	Sun May 26 12:12:07 2013 +0200
@@ -36,6 +36,7 @@
     LDAPAuthDB *authdb = malloc(sizeof (LDAPAuthDB));
     authdb->authdb.name = strdup(name);
     authdb->authdb.get_user = ldap_get_user;
+    authdb->authdb.use_cache = 1;
     authdb->config = *conf;
 
     if (!authdb->config.usersearch) {

--- a/src/server/daemon/objs.mk	Wed May 22 15:05:06 2013 +0200
+++ b/src/server/daemon/objs.mk	Sun May 26 12:12:07 2013 +0200
@@ -47,7 +47,7 @@
 DAEMONOBJ += log.o
 DAEMONOBJ += event.o
 DAEMONOBJ += threadpools.o
-DAEMONOBJ += authdb.o
+DAEMONOBJ += auth.o
 DAEMONOBJ += ldap_auth.o
 DAEMONOBJ += keyfile_auth.o
 DAEMONOBJ += error.o

--- a/src/server/daemon/protocol.c	Wed May 22 15:05:06 2013 +0200
+++ b/src/server/daemon/protocol.c	Sun May 26 12:12:07 2013 +0200
@@ -323,7 +323,7 @@
         
         // set stream property
         NetIOStream *stream = (NetIOStream*)sn->csd;
-        stream->chunkedenc = 1;
+        stream->chunked_enc = 1;
     }
     
     // add header from rq->srvhdrs

--- a/src/server/daemon/vfs.c	Wed May 22 15:05:06 2013 +0200
+++ b/src/server/daemon/vfs.c	Sun May 26 12:12:07 2013 +0200
@@ -41,6 +41,7 @@
 static VFS_IO sys_file_io = {
     sys_file_read,
     sys_file_write,
+    sys_file_seek,
     sys_file_close
 };
 
@@ -449,6 +450,10 @@
     return write(fd->fd, buf, nbyte);
 }
 
+off_t sys_file_seek(SYS_FILE fd, off_t offset, int whence) {
+    return lseek(fd->fd, offset, whence);
+}
+
 void sys_file_close(SYS_FILE fd) {
     close(fd->fd);
 }
@@ -506,6 +511,10 @@
     return fd->io->write(fd, buf, nbyte);
 }
 
+NSAPI_PUBLIC off_t system_lseek(SYS_FILE fd, off_t offset, int whence) {
+    return fd->io->seek(fd, offset, whence);
+}
+
 NSAPI_PUBLIC int system_fclose(SYS_FILE fd) {
     vfs_close(fd);
     return 0;

--- a/src/server/daemon/vfs.h	Wed May 22 15:05:06 2013 +0200
+++ b/src/server/daemon/vfs.h	Sun May 26 12:12:07 2013 +0200
@@ -47,6 +47,7 @@
 void sys_set_error_status(VFSContext *ctx);
 ssize_t sys_file_read(SYS_FILE fd, void *buf, size_t nbyte);
 ssize_t sys_file_write(SYS_FILE fd, const void *buf, size_t nbyte);
+off_t sys_file_seek(SYS_FILE fd, off_t offset, int whence);
 void sys_file_close(SYS_FILE fd);
 int sys_dir_read(VFS_DIR dir, VFS_ENTRY *entry, int getstat);
 void sys_dir_close(VFS_DIR dir);

--- a/src/server/daemon/webserver.c	Wed May 22 15:05:06 2013 +0200
+++ b/src/server/daemon/webserver.c	Sun May 26 12:12:07 2013 +0200
@@ -50,7 +50,7 @@
 #include "httplistener.h"
 #include "webserver.h"
 #include "log.h"
-
+#include "auth.h"
 
 extern struct FuncStruct webserver_funcs[];
 
@@ -75,6 +75,9 @@
         return -1;
     }
     
+    // init caches
+    auth_cache_init();
+    
     // create tmp dir and pid file
     ServerConfiguration *cfg = cfgmgr_get_server_config();
     char *mkdir_cmd = NULL;
@@ -140,7 +143,7 @@
                     LOG_WARN,
                     "server must be started as root to change uid");
     }
-   
+    
     return 0;
 }
 

--- a/src/server/public/auth.h	Wed May 22 15:05:06 2013 +0200
+++ b/src/server/public/auth.h	Sun May 26 12:12:07 2013 +0200
@@ -50,6 +50,7 @@
     char                *name;
     /* User* get_user(AuthDB *db, char *username) */
     authdb_get_user_f   get_user;
+    int                 use_cache;
 };
 
 /*
@@ -87,7 +88,8 @@
 };
 
 
-
+User* authdb_get_user(AuthDB *db, char *user);
+User* authdb_get_and_verify(AuthDB *db, char *user, char *password, int *pw);
 
 #ifdef	__cplusplus
 }

--- a/src/server/public/nsapi.h	Wed May 22 15:05:06 2013 +0200
+++ b/src/server/public/nsapi.h	Sun May 26 12:12:07 2013 +0200
@@ -1376,6 +1376,7 @@
 /* file */
 NSAPI_PUBLIC int system_fread(SYS_FILE fd, void *buf, int nbyte);
 NSAPI_PUBLIC int system_fwrite(SYS_FILE fd, const void *buf, int nbyte);
+NSAPI_PUBLIC off_t system_lseek(SYS_FILE fd, off_t offset, int whence);
 NSAPI_PUBLIC int system_fclose(SYS_FILE fd);
 
 NSAPI_PUBLIC int util_errno2status(int errno_value); // new

--- a/src/server/public/vfs.h	Wed May 22 15:05:06 2013 +0200
+++ b/src/server/public/vfs.h	Sun May 26 12:12:07 2013 +0200
@@ -89,6 +89,7 @@
 struct VFS_IO {
     ssize_t (*read)(SYS_FILE fd, void *buf, size_t nbyte);
     ssize_t (*write)(SYS_FILE fd, const void *buf, size_t nbyte);
+    off_t (*seek)(SYS_FILE fd, off_t offset, int whence);
     void (*close)(SYS_FILE fd);
 };
 

--- a/src/server/ucx/map.c	Wed May 22 15:05:06 2013 +0200
+++ b/src/server/ucx/map.c	Sun May 26 12:12:07 2013 +0200
@@ -1,5 +1,29 @@
 /*
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
  *
+ * Copyright 2013 Olaf Wintermann. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   1. Redistributions of source code must retain the above copyright
+ *      notice, this list of conditions and the following disclaimer.
+ *
+ *   2. Redistributions in binary form must reproduce the above copyright
+ *      notice, this list of conditions and the following disclaimer in the
+ *      documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
  */
 
 #include <stdlib.h>
@@ -28,7 +52,7 @@
     return map;
 }
 
-void ucx_map_free(UcxMap *map) {
+void ucx_map_free_elmlist(UcxMap *map) {
     for (size_t n = 0 ; n < map->size ; n++) {
         UcxMapElement *elem = map->map[n];
         if (elem != NULL) {
@@ -41,10 +65,15 @@
         }
     }
     free(map->map);
+}
+
+void ucx_map_free(UcxMap *map) {
+    ucx_map_free_elmlist(map);
     free(map);
 }
 
-int ucx_map_copy(UcxMap *from, UcxMap *to, copy_func fnc, void *data) {
+int ucx_map_copy(UcxMap *restrict from, UcxMap *restrict to,
+        copy_func fnc, void *data) {
     UcxMapIterator i = ucx_map_iterator(from);
     void *value;
     UCX_MAP_FOREACH(value, i) {
@@ -82,6 +111,9 @@
         }
         map->count = 0;
         ucx_map_copy(&oldmap, map, NULL, NULL);
+        
+        /* free the UcxMapElement list of oldmap */
+        ucx_map_free_elmlist(&oldmap);
     }
     return 0;
 }
@@ -92,8 +124,8 @@
     }
 
     size_t slot = key.hash%map->size;
-    UcxMapElement *elm = map->map[slot];
-    UcxMapElement *prev = NULL;
+    UcxMapElement *restrict elm = map->map[slot];
+    UcxMapElement *restrict prev = NULL;
 
     while (elm != NULL && elm->key.hash < key.hash) {
         prev = elm;
@@ -106,10 +138,10 @@
             return -1;
         }
         e->key.data = NULL;
-        if (prev == NULL) {
+        if (prev) {
+            prev->next = e;
+        } else {
             map->map[slot] = e;
-        } else {
-            prev->next = e;
         }
         e->next = elm;
         elm = e;
@@ -130,35 +162,57 @@
     return 0;
 }
 
-void* ucx_map_get(UcxMap *map, UcxKey key) {
+void* ucx_map_get_and_remove(UcxMap *map, UcxKey key, _Bool remove) {
     if(key.hash == 0) {
         key.hash = ucx_hash((char*)key.data, key.len);
     }
     
-    UcxMapElement *elm = map->map[key.hash%map->size];
-    while (elm != NULL && elm->key.hash <= key.hash) {
+    size_t slot = key.hash%map->size;
+    UcxMapElement *restrict elm = map->map[slot];
+    UcxMapElement *restrict pelm = NULL;
+    while (elm && elm->key.hash <= key.hash) {
         if(elm->key.hash == key.hash) {
             int n = (key.len > elm->key.len) ? elm->key.len : key.len;
             if (memcmp(elm->key.data, key.data, n) == 0) {
-                return elm->data;
+                void *data = elm->data;
+                if (remove) {
+                    if (pelm) {
+                        pelm->next = elm->next;
+                    } else {
+                        map->map[slot] = elm->next;
+                    }
+                    free(elm);
+                    map->count--;
+                }
+
+                return data;
             }
         }
-        elm = elm->next;
+        pelm = elm;
+        elm = pelm->next;
     }
 
     return NULL;
 }
 
+void *ucx_map_get(UcxMap *map, UcxKey key) {
+    return ucx_map_get_and_remove(map, key, 0);
+}
+
+void *ucx_map_remove(UcxMap *map, UcxKey key) {
+    return ucx_map_get_and_remove(map, key, 1);
+}
+
 UcxKey ucx_key(void *data, size_t len) {
     UcxKey key;
     key.data = data;
     key.len = len;
-    key.hash = ucx_hash(data, len);
+    key.hash = ucx_hash((const char*) data, len);
     return key;
 }
 
 
-int ucx_hash(char *data, size_t len) {
+int ucx_hash(const char *data, size_t len) {
     /* murmur hash 2 */
 
     int m = 0x5bd1e995;
@@ -257,13 +311,13 @@
 
         /* read into key buffer */
         n = 16;
-        key = malloc(n);
+        key = (char*) malloc(n);
         r = 0;
         do {
             if (c == '=') break;
             if (r > n - 2) {
                 n *= 2;
-                key = realloc(key, n);
+                key = (char*) realloc(key, n);
             }
             key[r] = c;
             r++;
@@ -286,13 +340,13 @@
 
         /* read into value buffer */
         n = 64;
-        value = malloc(n);
+        value = (char*) malloc(n);
         r = 0;
         do {
             if (c == '\n') break;
             if (r > n - 2) {
                 n *= 2;
-                value = realloc(value, n);
+                value = (char*) realloc(value, n);
             }
             value[r] = c;
             r++;
@@ -304,18 +358,18 @@
             size_t decodedSize;
             void *decoded = decoder(value, decdata, &decodedSize);
             free(value);
-            value = decoded;
+            value = (char*) decoded;
             r = decodedSize;
         } else {
             r += 2;
-            value = realloc(value, r);
+            value = (char*) realloc(value, r);
         }
 
         if (allocator.pool) {
             void *pooledValue = allocator.malloc(allocator.pool, r);
             memcpy(pooledValue, value, r);
             free(value);
-            value = pooledValue;
+            value = (char*) pooledValue;
         }
 
         ucx_map_cstr_put(map, key, value);
@@ -330,15 +384,15 @@
     UcxMapIterator iter = ucx_map_iterator(map);
     char *k, *v;
     sstr_t key, value;
-    int written;
+    size_t written;
 
     UCX_MAP_FOREACH(v, iter) {
         k = (char*) iter.cur->key.data;
-        key = sstr(k);
+        key = sstrn(k, iter.cur->key.len);
         if (encoder) {
             size_t encodedSize;
             void *encoded = encoder(v, encdata, &encodedSize);
-            value = sstrn(encoded,encodedSize - 1);
+            value = sstrn((char*) encoded,encodedSize - 1);
         } else {
             value = sstr(v);
         }

--- a/src/server/ucx/map.h	Wed May 22 15:05:06 2013 +0200
+++ b/src/server/ucx/map.h	Sun May 26 12:12:07 2013 +0200
@@ -1,5 +1,29 @@
 /*
- * 
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ *
+ * Copyright 2013 Olaf Wintermann. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   1. Redistributions of source code must retain the above copyright
+ *      notice, this list of conditions and the following disclaimer.
+ *
+ *   2. Redistributions in binary form must reproduce the above copyright
+ *      notice, this list of conditions and the following disclaimer in the
+ *      documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
  */
 
 #ifndef MAP_H
@@ -15,7 +39,7 @@
 #endif
 
 #define UCX_MAP_FOREACH(elm,iter) \
-        for(;ucx_map_iter_next(&iter,(void*)&elm)==0;)
+        for(;ucx_map_iter_next(&iter,(void**)&elm)==0;)
 
 typedef struct UcxMap          UcxMap;
 typedef struct UcxKey          UcxKey;
@@ -37,7 +61,7 @@
 };
 
 struct UcxKey {
-    void   *data;
+    char   *data;
     size_t len;
     int    hash;
 };
@@ -51,28 +75,46 @@
 struct UcxMapIterator {
     UcxMap        *map;
     UcxMapElement *cur;
-    int           index;
+    size_t        index;
 };
 
 
 UcxMap *ucx_map_new(size_t size);
 void ucx_map_free(UcxMap *map);
 /* you cannot clone maps with more than 390 mio entries */
-int ucx_map_copy(UcxMap *from, UcxMap *to, copy_func fnc, void *data);
+int ucx_map_copy(UcxMap *restrict from, UcxMap *restrict to,
+        copy_func fnc, void *data);
 UcxMap *ucx_map_clone(UcxMap *map, copy_func fnc, void *data);
 int ucx_map_rehash(UcxMap *map);
 
 int ucx_map_put(UcxMap *map, UcxKey key, void *data);
 void* ucx_map_get(UcxMap *map, UcxKey key);
+void* ucx_map_remove(UcxMap *map, UcxKey key);
 
-#define ucx_map_sstr_put(m, s, d) ucx_map_put(m, ucx_key(s.ptr, s.length), d)
-#define ucx_map_cstr_put(m, s, d) ucx_map_put(m, ucx_key(s, strlen(s)), d)
-#define ucx_map_sstr_get(m, s) ucx_map_get(m, ucx_key(s.ptr, s.length))
-#define ucx_map_cstr_get(m, s) ucx_map_get(m, ucx_key(s, strlen(s)))
+#define ucx_map_sstr_put(m, s, d) \
+    ucx_map_put(m, ucx_key(s.ptr, s.length), d)
+#define ucx_map_cstr_put(m, s, d) \
+    ucx_map_put(m, ucx_key((void*)s, strlen(s)), d)
+#define ucx_map_int_put(m, i, d) \
+    ucx_map_put(m, ucx_key((void*)&i, sizeof(i)), d)
+
+#define ucx_map_sstr_get(m, s) \
+    ucx_map_get(m, ucx_key(s.ptr, s.length))
+#define ucx_map_cstr_get(m, s) \
+    ucx_map_get(m, ucx_key((void*)s, strlen(s)))
+#define ucx_map_int_get(m, i) \
+    ucx_map_get(m, ucx_key((void*)&i, sizeof(int)))
+
+#define ucx_map_sstr_remove(m, s) \
+    ucx_map_remove(m, ucx_key(s.ptr, s.length))
+#define ucx_map_cstr_remove(m, s) \
+    ucx_map_remove(m, ucx_key((void*)s, strlen(s)))
+#define ucx_map_int_remove(m, i) \
+    ucx_map_remove(m, ucx_key((void*)&i, sizeof(i)))
 
 UcxKey ucx_key(void *data, size_t len);
 
-int ucx_hash(char *data, size_t len);
+int ucx_hash(const char *data, size_t len);
 
 UcxMapIterator ucx_map_iterator(UcxMap *map);
 

--- a/src/server/ucx/ucx.h	Wed May 22 15:05:06 2013 +0200
+++ b/src/server/ucx/ucx.h	Sun May 26 12:12:07 2013 +0200
@@ -1,8 +1,29 @@
-/* 
- * File:   ucx.h
- * Author: olaf
+/*
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ *
+ * Copyright 2013 Olaf Wintermann. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   1. Redistributions of source code must retain the above copyright
+ *      notice, this list of conditions and the following disclaimer.
  *
- * Created on 31. Dezember 2011, 17:17
+ *   2. Redistributions in binary form must reproduce the above copyright
+ *      notice, this list of conditions and the following disclaimer in the
+ *      documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
  */
 
 #ifndef UCX_H
@@ -11,18 +32,28 @@
 #include <stdlib.h>
 
 #ifdef	__cplusplus
+#ifndef _Bool
+#define _Bool bool
+#define restrict
+#endif
 extern "C" {
 #endif
 
 #define UCX_FOREACH(type,list,elem) \
         for (type elem = list ; elem != NULL ; elem = elem->next)
-    
+
 /* element1,element2,custom data -> {-1,0,1} */
 typedef int(*cmp_func)(void*,void*,void*);
 
 /* element,custom data -> copy of element */
 typedef void*(*copy_func)(void*,void*);
 
+/* buffer, element size, element count, stream */
+typedef size_t(*write_func)(const void*, size_t, size_t, void*);
+
+/* buffer, element size, element count, stream */
+typedef size_t(*read_func)(void*, size_t, size_t, void*);
+
 #ifdef	__cplusplus
 }
 #endif

--- a/src/server/util/io.c	Wed May 22 15:05:06 2013 +0200
+++ b/src/server/util/io.c	Sun May 26 12:12:07 2013 +0200
@@ -80,14 +80,14 @@
     st->st = net_io_funcs;
     st->fd = fd;
     st->max_read = 0;
-    st->rd = 0;
-    st->chunkedenc = 0;
+    st->read = 0;
+    st->chunked_enc = 0;
     st->buffered = 0;
     return (IOStream*)st;
 }
 
 ssize_t net_stream_write(NetIOStream *st, void *buf, size_t nbytes) {
-    if(st->chunkedenc) {
+    if(st->chunked_enc) {
         // TODO: on some plattforms iov_len is smaller than size_t
         struct iovec io[2];
         char chunk_len[16];
@@ -103,7 +103,7 @@
 }
 
 ssize_t net_stream_writev(NetIOStream *st, struct iovec *iovec, int iovcnt) {
-    if(st->chunkedenc) {
+    if(st->chunked_enc) {
         struct iovec *io = calloc(iovcnt + 1, sizeof(struct iovec));
         char chunk_len[16];
         io[0].iov_base = chunk_len;
@@ -121,11 +121,11 @@
 }
 
 ssize_t net_stream_read(NetIOStream *st, void *buf, size_t nbytes) {
-    if(st->max_read != 0 && st->rd >= st->max_read) {
+    if(st->max_read != 0 && st->read >= st->max_read) {
         return 0;
     }
     ssize_t r = read(st->fd, buf, nbytes);
-    st->rd += r;
+    st->read += r;
     return r;
 }
 
@@ -179,14 +179,75 @@
 }
 
 ssize_t net_sendfile(SYS_NETFD fd, sendfiledata *sfd) {
-    IOStream *out = fd;
-    if(out->sendfile) {
+    NetIOStream *out = fd;
+    if(out->st.sendfile && sfd->fd && sfd->fd->fd != -1) {
         ssize_t r = ((IOStream*)fd)->sendfile(fd, sfd);
         if(r < 0) {
             return IO_ERROR;
         }
     } else {
-        fprintf(stderr, "stream does not support sendfile\n");
+        // stream/file does not support sendfile
+        // do regular copy
+        char *buf = malloc(4096);
+        if(!buf) {
+            // TODO: out of memory error
+            return IO_ERROR;
+        }
+        char *header = (char*)sfd->header;
+        int hlen = sfd->hlen;
+        char *trailer = (char*)sfd->trailer;
+        int tlen = sfd->tlen;
+        if(header == NULL) {
+            hlen = 0;
+        }
+        if(trailer == NULL) {
+            tlen = 0;
+        }
+        
+        ssize_t r;
+        while(hlen > 0) {
+            r = out->st.write(fd, header, hlen);
+            header += r;
+            hlen -= r;
+            if(r <= 0) {
+                free(buf);
+                return IO_ERROR;
+            }
+        }
+        
+        if(system_lseek(sfd->fd, sfd->offset, SEEK_SET) == -1) {
+            free(buf);
+            return IO_ERROR;
+        }
+        
+        size_t length = sfd->len;
+        while(length > 0) {
+            if((r = system_fread(sfd->fd, buf, 4096)) <= 0) {
+                break;
+            }
+            char *write_buf = buf;
+            while(r > 0) {
+                ssize_t w = out->st.write(fd, write_buf, r);
+                r -= w;
+                length -= w;
+                write_buf += w;
+            }
+        }
+        free(buf);
+        if(length > 0) {
+            return IO_ERROR;
+        }
+        
+        while(tlen > 0) {
+            r = out->st.write(fd, trailer, tlen);
+            trailer += r;
+            tlen -= r;
+            if(r <= 0) {
+                return IO_ERROR;
+            }
+        }
+        
+        return sfd->hlen + sfd->len + sfd->tlen;
     }
     return IO_ERROR;
 }

--- a/src/server/util/io.h	Wed May 22 15:05:06 2013 +0200
+++ b/src/server/util/io.h	Sun May 26 12:12:07 2013 +0200
@@ -58,8 +58,8 @@
     IOStream st;
     int      fd;
     size_t   max_read;
-    size_t   rd;
-    int      chunkedenc;
+    size_t   read;
+    int      chunked_enc;
     int      buffered;
 } NetIOStream;