# HG changeset patch # User Olaf Wintermann # Date 1369563127 -7200 # Node ID 74babc0082b710590a89c8f140097c4c922bf117 # Parent 14722c5f8856268f3b28e1847ce097015859d93a added authentication cache diff -r 14722c5f8856 -r 74babc0082b7 src/server/daemon/acl.c --- a/src/server/daemon/acl.c Wed May 22 15:05:06 2013 +0200 +++ b/src/server/daemon/acl.c Sun May 26 12:12:07 2013 +0200 @@ -105,14 +105,10 @@ char *usr; char *pw; if(!basicauth_getuser(sn, rq, &usr, &pw)) { - user = list->defaultauthdb->get_user(list->defaultauthdb, usr); + int pwok; + user = authdb_get_and_verify(list->defaultauthdb, usr, pw, &pwok); if(!user) { - // wrong user name - return NULL; - } - if(!user->verify_password(user, pw)) { - // wrong password - user->free(user); + // wrong user or wrong password return NULL; } // ok - user is authenticated diff -r 14722c5f8856 -r 74babc0082b7 src/server/daemon/auth.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/server/daemon/auth.c Sun May 26 12:12:07 2013 +0200 @@ -0,0 +1,329 @@ +/* + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. + * + * Copyright 2013 Olaf Wintermann. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include "../ucx/map.h" +#include "../util/atomic.h" +#include "auth.h" + +static pthread_mutex_t auth_cache_mutex = PTHREAD_MUTEX_INITIALIZER; +static UserCache cache; + +void auth_cache_init() { + printf("auth_cache_init\n"); + // TODO: config parameters + //pthread_mutex_init(&auth_cache_mutex, NULL); + cache.map = calloc(64, sizeof(UserCacheElm)); + cache.size = 96; + cache.count = 0; + cache.max_users = 64; + cache.head = NULL; + cache.trail = NULL; +} + +User* auth_cache_get(char *authdb, char *user) { + //printf("auth_cache_get: %s\n", user); + /* + * create the key to access the map + * key: authdb\0user + */ + size_t authdblen = strlen(authdb); + size_t userlen = strlen(user); + + size_t keylen = authdblen + userlen + 1; + char *key = malloc(keylen); + memcpy(key, authdb, authdblen); + key[authdblen] = 0; + memcpy(key + authdblen + 1, user, userlen); + + UcxKey mapkey = ucx_key(key, keylen); + + // get cached user from map + time_t now = time(NULL); + size_t slot = mapkey.hash%cache.size; + + pthread_mutex_lock(&auth_cache_mutex); + + UserCacheElm *elm = cache.map[slot]; + while(elm && elm->key.hash != mapkey.hash) { + elm = elm->next_elm; + } + // if we have an elm, the hash is correct + if(elm) { + // compare the key data to be sure it is the correct user + int n = (mapkey.len > elm->key.len) ? elm->key.len : mapkey.len; + if (memcmp(elm->key.data, mapkey.data, n)) { + free(key); + pthread_mutex_unlock(&auth_cache_mutex); + return NULL; + } + } else { + free(key); + pthread_mutex_unlock(&auth_cache_mutex); + return NULL; + } + + // elm is now the correct UserCacheElm + // TODO: use configuration for expire time + if(now - elm->created > 120) { + // cached user expired + // remove all users from the list from the first to this one + UserCacheElm *e = cache.head; + while(e) { + if(e == elm) { + break; + } + UserCacheElm *n = e->next_user; + auth_cache_remove_from_map(e); + e = n; + } + cache.head = elm->next_user; + if(cache.trail == elm) { + cache.trail = NULL; + } + auth_cache_remove_from_map(elm); + free(key); + pthread_mutex_unlock(&auth_cache_mutex); + return NULL; + } + + pthread_mutex_unlock(&auth_cache_mutex); + free(key); + return (User*)elm->user; +} + +void auth_cache_add( + char *authdb, + User *user, + char *password, + char **groups, + size_t numgroups) +{ + //printf("auth_cache_add: %s\n", user->name); + /* + * this function does not check, if the user is already in the map + * use it only after auth_cache_get + */ + + CachedUser *cusr = malloc(sizeof(CachedUser)); + cusr->user.name = strdup(user->name); + cusr->user.uid = user->uid; + cusr->user.gid = user->gid; + cusr->user.verify_password = + (user_verify_passwd_f)cached_user_verify_password; + cusr->user.check_group = (user_check_group_f)cached_user_check_group; + cusr->user.free = (user_free_f)cached_user_unref; + + cusr->authdb = strdup(authdb); + cusr->password = strdup(password); + cusr->groups = calloc(numgroups, sizeof(sstr_t)); + cusr->numgroups = numgroups; + for(int i=0;igroups[i] = sstrdup(sstr(groups[i])); + } + cusr->ref = 1; + + /* + * add the user to the auth cache + * the auth cache is a list of all cached users + */ + + // create list element + time_t now = time(NULL); + UserCacheElm *elm = malloc(sizeof(UserCacheElm)); + elm->user = cusr; + elm->created = now; + elm->next_elm = NULL; + elm->next_user = NULL; + + // create map key + size_t authdblen = strlen(authdb); + size_t userlen = strlen(user->name); + size_t keylen = authdblen + userlen + 1; + char *key = malloc(keylen); + memcpy(key, authdb, authdblen); + key[authdblen] = 0; + memcpy(key + authdblen + 1, user->name, userlen); + UcxKey mapkey = ucx_key(key, keylen); + + elm->key = mapkey; + elm->slot = mapkey.hash%cache.size; + + // add user to list and map + pthread_mutex_lock(&auth_cache_mutex); + + // remove the first cached user if expired or the cache is full + if(cache.head && + (cache.count >= cache.max_users || now-cache.head->created > 120)) + { + UserCacheElm *first = cache.head; + cache.head = first->next_user; + if(!cache.head) { + cache.trail = NULL; + } + auth_cache_remove_from_map(first); + } + + // add to map + UserCacheElm *prevelm = cache.map[elm->slot]; + if(prevelm) { + for(;;) { + if(!prevelm->next_elm) { + break; + } + prevelm = prevelm->next_elm; + } + } + if(prevelm) { + prevelm->next_elm = elm; + } else { + cache.map[elm->slot] = elm; + } + + // add to list + if(cache.head) { + cache.trail->next_user = elm; + cache.trail = elm; + } else { + cache.head = elm; + cache.trail = elm; + } + + cache.count++; + + pthread_mutex_unlock(&auth_cache_mutex); +} + +void auth_cache_remove_from_map(UserCacheElm *elm) { + UserCacheElm *prevelm = NULL; + UserCacheElm *e = cache.map[elm->slot]; + while(e) { + if(e == elm) { + break; + } else { + prevelm = e; + } + e = e->next_elm; + } + if(prevelm) { + prevelm->next_elm = elm->next_elm; + } else { + cache.map[elm->slot] = elm->next_elm; + } + + free(elm->key.data); + cached_user_unref(elm->user); + free(elm); + + cache.count--; +} + +int cached_user_verify_password(CachedUser *user, char *password) { + if(!strcmp(user->password, password)) { + return 1; + } else { + return 0; + } +} + +int cached_user_check_group(CachedUser *user, char *group) { + sstr_t grp = sstr(group); + for(int i=0;inumgroups;i++) { + if(!sstrcmp(user->groups[i], grp)) { + return 1; + } + } + return 0; +} + +void cached_user_unref(CachedUser *user) { + uint32_t ref = ws_atomic_dec32(&user->ref); + if(ref == 0) { + cached_user_delete(user); + } +} + +void cached_user_delete(CachedUser *user) { + free(user->user.name); + free(user->authdb); + free(user->password); + free(user->groups); + free(user); +} + + +/* + * public API + * from public/auth.h + */ + +User* authdb_get_user(AuthDB *db, char *user) { + if(db->use_cache) { + User *u = auth_cache_get(db->name, user); + if(u) { + return u; + } + } + return db->get_user(db, user); +} + +User* authdb_get_and_verify(AuthDB *db, char *user, char *password, int *pw) { + User *u = NULL; + // try getting the user from the cache + if(db->use_cache) { + u = auth_cache_get(db->name, user); + if(u) { + if(u->verify_password(u, password)) { + *pw = 1; + } else { + *pw = 0; + u->free(u); + u = NULL; + } + return u; + } + } + // user not cached + u = db->get_user(db, user); + if(u) { + if(u->verify_password(u, password)) { + if(db->use_cache) { + auth_cache_add(db->name, u, password, NULL, 0); + } + *pw = 1; + } else { + *pw = 0; + u->free(u); + u = NULL; + } + } + return u; +} diff -r 14722c5f8856 -r 74babc0082b7 src/server/daemon/auth.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/server/daemon/auth.h Sun May 26 12:12:07 2013 +0200 @@ -0,0 +1,89 @@ +/* + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. + * + * Copyright 2013 Olaf Wintermann. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef AUTH_H +#define AUTH_H + +#include +#include "../public/auth.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + User user; + char *authdb; + sstr_t *groups; + size_t numgroups; + char *password; + uint32_t ref; +} CachedUser; + +typedef struct user_cache_elm UserCacheElm; +struct user_cache_elm { + CachedUser *user; + UserCacheElm *next_user; // next elm in the cached user list + UcxKey key; // key to access this element + size_t slot; // slot in the map + UserCacheElm *next_elm; // next element in this map slot + time_t created; +}; + +typedef struct { + UserCacheElm **map; + size_t size; + size_t count; + size_t max_users; + UserCacheElm *head; + UserCacheElm *trail; +} UserCache; + +void auth_cache_init(); + +User* auth_cache_get(char *authdb, char *user); +void auth_cache_add( + char *authdb, + User *user, + char *password, + char **groups, + size_t numgroups); + +void auth_cache_remove_from_map(UserCacheElm *elm); + +int cached_user_verify_password(CachedUser *user, char *password); +int cached_user_check_group(CachedUser *user, char *group); +void cached_user_unref(CachedUser *user); +void cached_user_delete(CachedUser *user); + +#ifdef __cplusplus +} +#endif + +#endif /* AUTH_H */ + diff -r 14722c5f8856 -r 74babc0082b7 src/server/daemon/authdb.c --- a/src/server/daemon/authdb.c Wed May 22 15:05:06 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,34 +0,0 @@ -/* - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. - * - * Copyright 2013 Olaf Wintermann. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include - -#include "../ucx/map.h" -#include "../public/auth.h" - diff -r 14722c5f8856 -r 74babc0082b7 src/server/daemon/keyfile_auth.c --- a/src/server/daemon/keyfile_auth.c Wed May 22 15:05:06 2013 +0200 +++ b/src/server/daemon/keyfile_auth.c Sun May 26 12:12:07 2013 +0200 @@ -39,6 +39,7 @@ Keyfile* keyfile_new() { Keyfile *keyfile = malloc(sizeof(Keyfile)); keyfile->authdb.get_user = keyfile_get_user; + keyfile->authdb.use_cache = 0; keyfile->users = ucx_map_new(16); keyfile->ref = 1; return keyfile; diff -r 14722c5f8856 -r 74babc0082b7 src/server/daemon/keyfile_auth.h --- a/src/server/daemon/keyfile_auth.h Wed May 22 15:05:06 2013 +0200 +++ b/src/server/daemon/keyfile_auth.h Sun May 26 12:12:07 2013 +0200 @@ -52,7 +52,7 @@ struct keyfile_user { User user; sstr_t *groups; - size_t numgroups; + size_t numgroups; enum KeyfileHashType hash_type; char *hash; size_t hashlen; diff -r 14722c5f8856 -r 74babc0082b7 src/server/daemon/ldap_auth.c --- a/src/server/daemon/ldap_auth.c Wed May 22 15:05:06 2013 +0200 +++ b/src/server/daemon/ldap_auth.c Sun May 26 12:12:07 2013 +0200 @@ -36,6 +36,7 @@ LDAPAuthDB *authdb = malloc(sizeof (LDAPAuthDB)); authdb->authdb.name = strdup(name); authdb->authdb.get_user = ldap_get_user; + authdb->authdb.use_cache = 1; authdb->config = *conf; if (!authdb->config.usersearch) { diff -r 14722c5f8856 -r 74babc0082b7 src/server/daemon/objs.mk --- a/src/server/daemon/objs.mk Wed May 22 15:05:06 2013 +0200 +++ b/src/server/daemon/objs.mk Sun May 26 12:12:07 2013 +0200 @@ -47,7 +47,7 @@ DAEMONOBJ += log.o DAEMONOBJ += event.o DAEMONOBJ += threadpools.o -DAEMONOBJ += authdb.o +DAEMONOBJ += auth.o DAEMONOBJ += ldap_auth.o DAEMONOBJ += keyfile_auth.o DAEMONOBJ += error.o diff -r 14722c5f8856 -r 74babc0082b7 src/server/daemon/protocol.c --- a/src/server/daemon/protocol.c Wed May 22 15:05:06 2013 +0200 +++ b/src/server/daemon/protocol.c Sun May 26 12:12:07 2013 +0200 @@ -323,7 +323,7 @@ // set stream property NetIOStream *stream = (NetIOStream*)sn->csd; - stream->chunkedenc = 1; + stream->chunked_enc = 1; } // add header from rq->srvhdrs diff -r 14722c5f8856 -r 74babc0082b7 src/server/daemon/vfs.c --- a/src/server/daemon/vfs.c Wed May 22 15:05:06 2013 +0200 +++ b/src/server/daemon/vfs.c Sun May 26 12:12:07 2013 +0200 @@ -41,6 +41,7 @@ static VFS_IO sys_file_io = { sys_file_read, sys_file_write, + sys_file_seek, sys_file_close }; @@ -449,6 +450,10 @@ return write(fd->fd, buf, nbyte); } +off_t sys_file_seek(SYS_FILE fd, off_t offset, int whence) { + return lseek(fd->fd, offset, whence); +} + void sys_file_close(SYS_FILE fd) { close(fd->fd); } @@ -506,6 +511,10 @@ return fd->io->write(fd, buf, nbyte); } +NSAPI_PUBLIC off_t system_lseek(SYS_FILE fd, off_t offset, int whence) { + return fd->io->seek(fd, offset, whence); +} + NSAPI_PUBLIC int system_fclose(SYS_FILE fd) { vfs_close(fd); return 0; diff -r 14722c5f8856 -r 74babc0082b7 src/server/daemon/vfs.h --- a/src/server/daemon/vfs.h Wed May 22 15:05:06 2013 +0200 +++ b/src/server/daemon/vfs.h Sun May 26 12:12:07 2013 +0200 @@ -47,6 +47,7 @@ void sys_set_error_status(VFSContext *ctx); ssize_t sys_file_read(SYS_FILE fd, void *buf, size_t nbyte); ssize_t sys_file_write(SYS_FILE fd, const void *buf, size_t nbyte); +off_t sys_file_seek(SYS_FILE fd, off_t offset, int whence); void sys_file_close(SYS_FILE fd); int sys_dir_read(VFS_DIR dir, VFS_ENTRY *entry, int getstat); void sys_dir_close(VFS_DIR dir); diff -r 14722c5f8856 -r 74babc0082b7 src/server/daemon/webserver.c --- a/src/server/daemon/webserver.c Wed May 22 15:05:06 2013 +0200 +++ b/src/server/daemon/webserver.c Sun May 26 12:12:07 2013 +0200 @@ -50,7 +50,7 @@ #include "httplistener.h" #include "webserver.h" #include "log.h" - +#include "auth.h" extern struct FuncStruct webserver_funcs[]; @@ -75,6 +75,9 @@ return -1; } + // init caches + auth_cache_init(); + // create tmp dir and pid file ServerConfiguration *cfg = cfgmgr_get_server_config(); char *mkdir_cmd = NULL; @@ -140,7 +143,7 @@ LOG_WARN, "server must be started as root to change uid"); } - + return 0; } diff -r 14722c5f8856 -r 74babc0082b7 src/server/public/auth.h --- a/src/server/public/auth.h Wed May 22 15:05:06 2013 +0200 +++ b/src/server/public/auth.h Sun May 26 12:12:07 2013 +0200 @@ -50,6 +50,7 @@ char *name; /* User* get_user(AuthDB *db, char *username) */ authdb_get_user_f get_user; + int use_cache; }; /* @@ -87,7 +88,8 @@ }; - +User* authdb_get_user(AuthDB *db, char *user); +User* authdb_get_and_verify(AuthDB *db, char *user, char *password, int *pw); #ifdef __cplusplus } diff -r 14722c5f8856 -r 74babc0082b7 src/server/public/nsapi.h --- a/src/server/public/nsapi.h Wed May 22 15:05:06 2013 +0200 +++ b/src/server/public/nsapi.h Sun May 26 12:12:07 2013 +0200 @@ -1376,6 +1376,7 @@ /* file */ NSAPI_PUBLIC int system_fread(SYS_FILE fd, void *buf, int nbyte); NSAPI_PUBLIC int system_fwrite(SYS_FILE fd, const void *buf, int nbyte); +NSAPI_PUBLIC off_t system_lseek(SYS_FILE fd, off_t offset, int whence); NSAPI_PUBLIC int system_fclose(SYS_FILE fd); NSAPI_PUBLIC int util_errno2status(int errno_value); // new diff -r 14722c5f8856 -r 74babc0082b7 src/server/public/vfs.h --- a/src/server/public/vfs.h Wed May 22 15:05:06 2013 +0200 +++ b/src/server/public/vfs.h Sun May 26 12:12:07 2013 +0200 @@ -89,6 +89,7 @@ struct VFS_IO { ssize_t (*read)(SYS_FILE fd, void *buf, size_t nbyte); ssize_t (*write)(SYS_FILE fd, const void *buf, size_t nbyte); + off_t (*seek)(SYS_FILE fd, off_t offset, int whence); void (*close)(SYS_FILE fd); }; diff -r 14722c5f8856 -r 74babc0082b7 src/server/ucx/map.c --- a/src/server/ucx/map.c Wed May 22 15:05:06 2013 +0200 +++ b/src/server/ucx/map.c Sun May 26 12:12:07 2013 +0200 @@ -1,5 +1,29 @@ /* + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. * + * Copyright 2013 Olaf Wintermann. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. */ #include @@ -28,7 +52,7 @@ return map; } -void ucx_map_free(UcxMap *map) { +void ucx_map_free_elmlist(UcxMap *map) { for (size_t n = 0 ; n < map->size ; n++) { UcxMapElement *elem = map->map[n]; if (elem != NULL) { @@ -41,10 +65,15 @@ } } free(map->map); +} + +void ucx_map_free(UcxMap *map) { + ucx_map_free_elmlist(map); free(map); } -int ucx_map_copy(UcxMap *from, UcxMap *to, copy_func fnc, void *data) { +int ucx_map_copy(UcxMap *restrict from, UcxMap *restrict to, + copy_func fnc, void *data) { UcxMapIterator i = ucx_map_iterator(from); void *value; UCX_MAP_FOREACH(value, i) { @@ -82,6 +111,9 @@ } map->count = 0; ucx_map_copy(&oldmap, map, NULL, NULL); + + /* free the UcxMapElement list of oldmap */ + ucx_map_free_elmlist(&oldmap); } return 0; } @@ -92,8 +124,8 @@ } size_t slot = key.hash%map->size; - UcxMapElement *elm = map->map[slot]; - UcxMapElement *prev = NULL; + UcxMapElement *restrict elm = map->map[slot]; + UcxMapElement *restrict prev = NULL; while (elm != NULL && elm->key.hash < key.hash) { prev = elm; @@ -106,10 +138,10 @@ return -1; } e->key.data = NULL; - if (prev == NULL) { + if (prev) { + prev->next = e; + } else { map->map[slot] = e; - } else { - prev->next = e; } e->next = elm; elm = e; @@ -130,35 +162,57 @@ return 0; } -void* ucx_map_get(UcxMap *map, UcxKey key) { +void* ucx_map_get_and_remove(UcxMap *map, UcxKey key, _Bool remove) { if(key.hash == 0) { key.hash = ucx_hash((char*)key.data, key.len); } - UcxMapElement *elm = map->map[key.hash%map->size]; - while (elm != NULL && elm->key.hash <= key.hash) { + size_t slot = key.hash%map->size; + UcxMapElement *restrict elm = map->map[slot]; + UcxMapElement *restrict pelm = NULL; + while (elm && elm->key.hash <= key.hash) { if(elm->key.hash == key.hash) { int n = (key.len > elm->key.len) ? elm->key.len : key.len; if (memcmp(elm->key.data, key.data, n) == 0) { - return elm->data; + void *data = elm->data; + if (remove) { + if (pelm) { + pelm->next = elm->next; + } else { + map->map[slot] = elm->next; + } + free(elm); + map->count--; + } + + return data; } } - elm = elm->next; + pelm = elm; + elm = pelm->next; } return NULL; } +void *ucx_map_get(UcxMap *map, UcxKey key) { + return ucx_map_get_and_remove(map, key, 0); +} + +void *ucx_map_remove(UcxMap *map, UcxKey key) { + return ucx_map_get_and_remove(map, key, 1); +} + UcxKey ucx_key(void *data, size_t len) { UcxKey key; key.data = data; key.len = len; - key.hash = ucx_hash(data, len); + key.hash = ucx_hash((const char*) data, len); return key; } -int ucx_hash(char *data, size_t len) { +int ucx_hash(const char *data, size_t len) { /* murmur hash 2 */ int m = 0x5bd1e995; @@ -257,13 +311,13 @@ /* read into key buffer */ n = 16; - key = malloc(n); + key = (char*) malloc(n); r = 0; do { if (c == '=') break; if (r > n - 2) { n *= 2; - key = realloc(key, n); + key = (char*) realloc(key, n); } key[r] = c; r++; @@ -286,13 +340,13 @@ /* read into value buffer */ n = 64; - value = malloc(n); + value = (char*) malloc(n); r = 0; do { if (c == '\n') break; if (r > n - 2) { n *= 2; - value = realloc(value, n); + value = (char*) realloc(value, n); } value[r] = c; r++; @@ -304,18 +358,18 @@ size_t decodedSize; void *decoded = decoder(value, decdata, &decodedSize); free(value); - value = decoded; + value = (char*) decoded; r = decodedSize; } else { r += 2; - value = realloc(value, r); + value = (char*) realloc(value, r); } if (allocator.pool) { void *pooledValue = allocator.malloc(allocator.pool, r); memcpy(pooledValue, value, r); free(value); - value = pooledValue; + value = (char*) pooledValue; } ucx_map_cstr_put(map, key, value); @@ -330,15 +384,15 @@ UcxMapIterator iter = ucx_map_iterator(map); char *k, *v; sstr_t key, value; - int written; + size_t written; UCX_MAP_FOREACH(v, iter) { k = (char*) iter.cur->key.data; - key = sstr(k); + key = sstrn(k, iter.cur->key.len); if (encoder) { size_t encodedSize; void *encoded = encoder(v, encdata, &encodedSize); - value = sstrn(encoded,encodedSize - 1); + value = sstrn((char*) encoded,encodedSize - 1); } else { value = sstr(v); } diff -r 14722c5f8856 -r 74babc0082b7 src/server/ucx/map.h --- a/src/server/ucx/map.h Wed May 22 15:05:06 2013 +0200 +++ b/src/server/ucx/map.h Sun May 26 12:12:07 2013 +0200 @@ -1,5 +1,29 @@ /* - * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. + * + * Copyright 2013 Olaf Wintermann. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. */ #ifndef MAP_H @@ -15,7 +39,7 @@ #endif #define UCX_MAP_FOREACH(elm,iter) \ - for(;ucx_map_iter_next(&iter,(void*)&elm)==0;) + for(;ucx_map_iter_next(&iter,(void**)&elm)==0;) typedef struct UcxMap UcxMap; typedef struct UcxKey UcxKey; @@ -37,7 +61,7 @@ }; struct UcxKey { - void *data; + char *data; size_t len; int hash; }; @@ -51,28 +75,46 @@ struct UcxMapIterator { UcxMap *map; UcxMapElement *cur; - int index; + size_t index; }; UcxMap *ucx_map_new(size_t size); void ucx_map_free(UcxMap *map); /* you cannot clone maps with more than 390 mio entries */ -int ucx_map_copy(UcxMap *from, UcxMap *to, copy_func fnc, void *data); +int ucx_map_copy(UcxMap *restrict from, UcxMap *restrict to, + copy_func fnc, void *data); UcxMap *ucx_map_clone(UcxMap *map, copy_func fnc, void *data); int ucx_map_rehash(UcxMap *map); int ucx_map_put(UcxMap *map, UcxKey key, void *data); void* ucx_map_get(UcxMap *map, UcxKey key); +void* ucx_map_remove(UcxMap *map, UcxKey key); -#define ucx_map_sstr_put(m, s, d) ucx_map_put(m, ucx_key(s.ptr, s.length), d) -#define ucx_map_cstr_put(m, s, d) ucx_map_put(m, ucx_key(s, strlen(s)), d) -#define ucx_map_sstr_get(m, s) ucx_map_get(m, ucx_key(s.ptr, s.length)) -#define ucx_map_cstr_get(m, s) ucx_map_get(m, ucx_key(s, strlen(s))) +#define ucx_map_sstr_put(m, s, d) \ + ucx_map_put(m, ucx_key(s.ptr, s.length), d) +#define ucx_map_cstr_put(m, s, d) \ + ucx_map_put(m, ucx_key((void*)s, strlen(s)), d) +#define ucx_map_int_put(m, i, d) \ + ucx_map_put(m, ucx_key((void*)&i, sizeof(i)), d) + +#define ucx_map_sstr_get(m, s) \ + ucx_map_get(m, ucx_key(s.ptr, s.length)) +#define ucx_map_cstr_get(m, s) \ + ucx_map_get(m, ucx_key((void*)s, strlen(s))) +#define ucx_map_int_get(m, i) \ + ucx_map_get(m, ucx_key((void*)&i, sizeof(int))) + +#define ucx_map_sstr_remove(m, s) \ + ucx_map_remove(m, ucx_key(s.ptr, s.length)) +#define ucx_map_cstr_remove(m, s) \ + ucx_map_remove(m, ucx_key((void*)s, strlen(s))) +#define ucx_map_int_remove(m, i) \ + ucx_map_remove(m, ucx_key((void*)&i, sizeof(i))) UcxKey ucx_key(void *data, size_t len); -int ucx_hash(char *data, size_t len); +int ucx_hash(const char *data, size_t len); UcxMapIterator ucx_map_iterator(UcxMap *map); diff -r 14722c5f8856 -r 74babc0082b7 src/server/ucx/ucx.h --- a/src/server/ucx/ucx.h Wed May 22 15:05:06 2013 +0200 +++ b/src/server/ucx/ucx.h Sun May 26 12:12:07 2013 +0200 @@ -1,8 +1,29 @@ -/* - * File: ucx.h - * Author: olaf +/* + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. + * + * Copyright 2013 Olaf Wintermann. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. * - * Created on 31. Dezember 2011, 17:17 + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. */ #ifndef UCX_H @@ -11,18 +32,28 @@ #include #ifdef __cplusplus +#ifndef _Bool +#define _Bool bool +#define restrict +#endif extern "C" { #endif #define UCX_FOREACH(type,list,elem) \ for (type elem = list ; elem != NULL ; elem = elem->next) - + /* element1,element2,custom data -> {-1,0,1} */ typedef int(*cmp_func)(void*,void*,void*); /* element,custom data -> copy of element */ typedef void*(*copy_func)(void*,void*); +/* buffer, element size, element count, stream */ +typedef size_t(*write_func)(const void*, size_t, size_t, void*); + +/* buffer, element size, element count, stream */ +typedef size_t(*read_func)(void*, size_t, size_t, void*); + #ifdef __cplusplus } #endif diff -r 14722c5f8856 -r 74babc0082b7 src/server/util/io.c --- a/src/server/util/io.c Wed May 22 15:05:06 2013 +0200 +++ b/src/server/util/io.c Sun May 26 12:12:07 2013 +0200 @@ -80,14 +80,14 @@ st->st = net_io_funcs; st->fd = fd; st->max_read = 0; - st->rd = 0; - st->chunkedenc = 0; + st->read = 0; + st->chunked_enc = 0; st->buffered = 0; return (IOStream*)st; } ssize_t net_stream_write(NetIOStream *st, void *buf, size_t nbytes) { - if(st->chunkedenc) { + if(st->chunked_enc) { // TODO: on some plattforms iov_len is smaller than size_t struct iovec io[2]; char chunk_len[16]; @@ -103,7 +103,7 @@ } ssize_t net_stream_writev(NetIOStream *st, struct iovec *iovec, int iovcnt) { - if(st->chunkedenc) { + if(st->chunked_enc) { struct iovec *io = calloc(iovcnt + 1, sizeof(struct iovec)); char chunk_len[16]; io[0].iov_base = chunk_len; @@ -121,11 +121,11 @@ } ssize_t net_stream_read(NetIOStream *st, void *buf, size_t nbytes) { - if(st->max_read != 0 && st->rd >= st->max_read) { + if(st->max_read != 0 && st->read >= st->max_read) { return 0; } ssize_t r = read(st->fd, buf, nbytes); - st->rd += r; + st->read += r; return r; } @@ -179,14 +179,75 @@ } ssize_t net_sendfile(SYS_NETFD fd, sendfiledata *sfd) { - IOStream *out = fd; - if(out->sendfile) { + NetIOStream *out = fd; + if(out->st.sendfile && sfd->fd && sfd->fd->fd != -1) { ssize_t r = ((IOStream*)fd)->sendfile(fd, sfd); if(r < 0) { return IO_ERROR; } } else { - fprintf(stderr, "stream does not support sendfile\n"); + // stream/file does not support sendfile + // do regular copy + char *buf = malloc(4096); + if(!buf) { + // TODO: out of memory error + return IO_ERROR; + } + char *header = (char*)sfd->header; + int hlen = sfd->hlen; + char *trailer = (char*)sfd->trailer; + int tlen = sfd->tlen; + if(header == NULL) { + hlen = 0; + } + if(trailer == NULL) { + tlen = 0; + } + + ssize_t r; + while(hlen > 0) { + r = out->st.write(fd, header, hlen); + header += r; + hlen -= r; + if(r <= 0) { + free(buf); + return IO_ERROR; + } + } + + if(system_lseek(sfd->fd, sfd->offset, SEEK_SET) == -1) { + free(buf); + return IO_ERROR; + } + + size_t length = sfd->len; + while(length > 0) { + if((r = system_fread(sfd->fd, buf, 4096)) <= 0) { + break; + } + char *write_buf = buf; + while(r > 0) { + ssize_t w = out->st.write(fd, write_buf, r); + r -= w; + length -= w; + write_buf += w; + } + } + free(buf); + if(length > 0) { + return IO_ERROR; + } + + while(tlen > 0) { + r = out->st.write(fd, trailer, tlen); + trailer += r; + tlen -= r; + if(r <= 0) { + return IO_ERROR; + } + } + + return sfd->hlen + sfd->len + sfd->tlen; } return IO_ERROR; } diff -r 14722c5f8856 -r 74babc0082b7 src/server/util/io.h --- a/src/server/util/io.h Wed May 22 15:05:06 2013 +0200 +++ b/src/server/util/io.h Sun May 26 12:12:07 2013 +0200 @@ -58,8 +58,8 @@ IOStream st; int fd; size_t max_read; - size_t rd; - int chunkedenc; + size_t read; + int chunked_enc; int buffered; } NetIOStream;