Sat, 02 May 2015 10:59:02 +0200
improved error messages
/* * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. * * Copyright 2015 Olaf Wintermann. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include <stdio.h> #include <stdlib.h> #include <string.h> #include <errno.h> #include <unistd.h> #include <time.h> #include <libxml/xmlerror.h> #include <sys/types.h> #include <ucx/string.h> #include <ucx/utils.h> #include <dirent.h> #include <libidav/webdav.h> #include <libidav/utils.h> #include "config.h" #include "scfg.h" #include "sopt.h" #include "db.h" #include "sync.h" #include "ucx/properties.h" static DavContext *ctx; static void xmlerrorfnc(void * c, const char * msg, ... ) { // nothing } int main(int argc, char **argv) { if(argc < 2) { fprintf(stderr, "Missing command\n"); print_usage(argv[0]); return -1; } char *cmd = argv[1]; CmdArgs *args = cmd_parse_args(argc - 2, argv + 2); if(!args) { print_usage(argv[0]); return -1; } xmlGenericErrorFunc fnc = xmlerrorfnc; initGenericErrorDefaultFunc(&fnc); ctx = dav_context_new(); load_config(ctx); // copy proxy config memcpy(ctx->http_proxy, get_http_proxy(), sizeof(Proxy)); memcpy(ctx->https_proxy, get_https_proxy(), sizeof(Proxy)); if(load_sync_config()) { return EXIT_FAILURE; } int ret = EXIT_FAILURE; if(!strcmp(cmd, "pull")) { ret = cmd_pull(args); } else if(!strcmp(cmd, "push")) { ret = cmd_push(args); } // TODO: cleanup sync config (don't forget to call regfree for regex) return ret; } void print_usage(char *cmd) { fprintf(stderr, "Usage: %s command [options] arguments...\n\n", cmd); fprintf(stderr, "Commands:\n"); fprintf(stderr, " pull [-c] <directory>\n"); fprintf(stderr, " push [-r] <directory>\n\n"); fprintf(stderr, "Options:\n"); fprintf(stderr, " -c Disable conflict detection\n"); fprintf(stderr, " -r Read changes from stdin\n\n"); } static int res_matches_filter(SyncDirectory *dir, char *res_path) { // trash filter if (dir->trash) { sstr_t rpath = sstr(util_concat_path(dir->path, res_path)); if (sstrprefix(rpath, sstr(dir->trash))) { free(rpath.ptr); return 1; } free(rpath.ptr); } // include/exclude filter UCX_FOREACH(inc, dir->include) { regex_t* pattern = (regex_t*) inc->data; if (regexec(pattern, res_path, 0, NULL, 0) == 0) { UCX_FOREACH(exc, dir->exclude) { regex_t* pattern = (regex_t*) exc->data; if (regexec(pattern, res_path, 0, NULL, 0) == 0) { return 1; } } return 0; } } return 1; } static DavSession* create_session(DavContext *ctx, Repository *repo, char *url) { DavSession *sn = dav_session_new_auth( ctx, url, repo->user, repo->password); curl_easy_setopt(sn->handle, CURLOPT_SSLVERSION, repo->ssl_version); sn->flags = get_repository_flags(repo); sn->key = dav_context_get_key(ctx, repo->default_key); return sn; } int cmd_pull(CmdArgs *a) { if(a->argc != 1) { fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many"); return -1; } SyncDirectory *dir = scfg_get_dir(a->argv[0]); if(!dir) { fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]); return -1; } Repository *repo = get_repository(sstr(dir->repository)); if(!repo) { fprintf(stderr, "Unkown repository %s\n", dir->name); return -1; } SyncDatabase *db = load_db(dir->database); if(!db) { fprintf(stderr, "Cannot load database file: %s\n", dir->database); return -1; } char *new_url = NULL; if(dir->collection) { new_url = util_concat_path(repo->url, dir->collection); } DavSession *sn = create_session(ctx, repo, new_url ? new_url : repo->url); if(new_url) { free(new_url); } if (cmd_getoption(a, "verbose")) { curl_easy_setopt(sn->handle, CURLOPT_VERBOSE, 1L); curl_easy_setopt(sn->handle, CURLOPT_STDERR, stderr); } DavResource *ls = dav_query(sn, "get D:getetag from / where lastmodified > 0 with depth -1"); if(!ls) { fprintf(stderr, "Error\n"); // TODO: free return -1; } if(!ls->children) { // TODO: free return 0; // empty repository } UcxMap *svrres = ucx_map_new(db->resources->count); UcxList *stack = ucx_list_prepend(NULL, ls->children); while(stack) { DavResource *res = stack->data; stack = ucx_list_remove(stack, stack); while(res) { if (res_matches_filter(dir, res->path)) { res = res->next; continue; } // download the resource if(sync_get_resource(a, dir, res, db)) { fprintf(stderr, "sync_get_resource failed for resource: %s\n", res->path); } // add every resource from the server to svrres // then db-resources only contains resources which are not on the // server LocalResource *local = ucx_map_cstr_get(db->resources, res->path); ucx_map_cstr_put(svrres, res->path, local); ucx_map_cstr_remove(db->resources, res->path); if(res->children) { stack = ucx_list_prepend(stack, res->children); } res = res->next; } } // delete every remotely removed resource UcxMapIterator i = ucx_map_iterator(db->resources); LocalResource *local; UCX_MAP_FOREACH(key, local, i) { if (res_matches_filter(dir, local->path)) { continue; } // sync_remove_resource does all necessary tests sync_remove_local_resource(dir, local); } ucx_map_free(db->resources); db->resources = svrres; // TODO: cleanup - BUT DONT CLEANUP SYNC CONFIG (do this in main!) // store db if(store_db(db, dir->database)) { fprintf(stderr, "Cannot store sync db\n"); return -1; } return 0; } int sync_get_resource(CmdArgs *a, SyncDirectory *dir, DavResource *res, SyncDatabase *db) { int cdt = cmd_getoption(a, "conflict") ? 0 : 1; // conflict detection LocalResource *local = ucx_map_cstr_get(db->resources, res->path); char *local_path = util_concat_path(dir->path, res->path); char *etag = dav_get_property(res, "D:getetag"); struct stat s; if(local) { int exists = 1; if(stat(local_path, &s)) { // Ignore the fact, that the file is locally removed. If the // server has an updated version, we readd the file or the // next push will delete it on the server. if(errno != ENOENT) { fprintf(stderr, "Cannot stat file: %s\n", local_path); free(local_path); return -1; } else { exists = 0; } } if(local->etag) { sstr_t e = sstr(etag); if(sstrprefix(e, S("W/"))) { e = sstrsubs(e, 2); } if(!strcmp(e.ptr, local->etag)) { // resource is already up-to-date on the client return 0; } } if(cdt && exists && s.st_mtime != local->last_modified) { // file modified on the server and on the client rename_local_file(dir, db, local->path); } } else { if(stat(local_path, &s)) { if(errno != ENOENT) { fprintf(stderr, "Cannot stat file: %s\n", local_path); } } else if(S_ISDIR(s.st_mode)) { //fprintf(stderr, "Error: file %s is a directory\n", local_path); } else if(cdt) { // rename file on conflict rename_local_file(dir, db, res->path); } } int ret = 0; if(res->iscollection) { mode_t mode = S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH; if(util_mkdir(local_path, mode) && errno != EEXIST) { ret = -1; } } else { FILE *out = fopen(local_path, "wb"); if(!out) { fprintf(stderr, "Cannot open output file: %s\n", local_path); free(local_path); return -1; } printf("get: %s\n", res->path); if(dav_get_content(res, out, (dav_write_func)fwrite)) { ret = -1; } fclose(out); if(stat(local_path, &s)) { fprintf(stderr, "Cannot stat file: %s\n", local_path); } if(ret == 0) { if(!local) { // new local resource local = calloc(1, sizeof(LocalResource)); local->path = strdup(res->path); ucx_map_cstr_put(db->resources, local->path, local); } if(local->etag) { free(local->etag); } // set metadata from stat local->etag = etag; local->last_modified = s.st_mtime; local->size = s.st_size; } } free(local_path); return ret; } void sync_remove_local_resource(SyncDirectory *dir, LocalResource *res) { char *local_path = util_concat_path(dir->path, res->path); struct stat s; if(stat(local_path, &s)) { free(local_path); return; } if(s.st_mtime != res->last_modified) { free(local_path); return; } printf("delete: %s\n", res->path); if(dir->trash) { move_to_trash(dir, local_path); } else if(unlink(local_path)) { fprintf(stderr, "Cannot remove file %s\n", local_path); } free(local_path); } void rename_local_file(SyncDirectory *dir, SyncDatabase *db, char *path) { char *local_path = util_concat_path(dir->path, path); char *parent = util_parent_path(local_path); int rev = 0; struct stat s; int loop = 1; do { sstr_t new_path = ucx_asprintf( ucx_default_allocator(), "%sorig.%d.%s", parent, rev, util_resource_name(path)); if(stat(new_path.ptr, &s)) { if(errno == ENOENT) { loop = 0; printf("conflict: %s\n", local_path); if(rename(local_path, new_path.ptr)) { //printf("errno: %d\n", errno); fprintf( stderr, "Cannot rename file %s to %s\n", local_path, new_path.ptr); } } } rev++; free(new_path.ptr); } while(loop); free(parent); } void move_to_trash(SyncDirectory *dir, char *path) { char *new_path = NULL; for (int i=0;;i++) { sstr_t np = ucx_asprintf( ucx_default_allocator(), "%s%d-%s", dir->trash, i, util_resource_name(path)); struct stat s; if(stat(np.ptr, &s)) { if(errno == ENOENT) { new_path = np.ptr; } break; } free(np.ptr); }; if(!new_path) { fprintf(stderr, "Cannot move file %s to trash.\n", path); return; } if(rename(path, new_path)) { //printf("errno: %d\n", errno); fprintf( stderr, "Cannot rename file %s to %s\n", path, new_path); } free(new_path); } int cmd_push(CmdArgs *a) { if(a->argc != 1) { fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many"); return -1; } SyncDirectory *dir = scfg_get_dir(a->argv[0]); if(!dir) { fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]); return -1; } Repository *repo = get_repository(sstr(dir->repository)); if(!repo) { fprintf(stderr, "Unkown repository %s\n", dir->name); return -1; } SyncDatabase *db = load_db(dir->database); if(!db) { fprintf(stderr, "Cannot load database file: %s\n", dir->database); return -1; } char *new_url = NULL; if(dir->collection) { new_url = util_concat_path(repo->url, dir->collection); } DavSession *sn = create_session(ctx, repo, new_url ? new_url : repo->url); if(new_url) { free(new_url); } if (cmd_getoption(a, "verbose")) { curl_easy_setopt(sn->handle, CURLOPT_VERBOSE, 1L); curl_easy_setopt(sn->handle, CURLOPT_STDERR, stderr); } // upload all changed files UcxList *resources = cmd_getoption(a, "read") ? read_changes(dir, db) : local_scan(dir, db); UcxMap *lclres = ucx_map_new(db->resources->count); UCX_FOREACH(elm, resources) { LocalResource *local_res = elm->data; if (!res_matches_filter(dir, local_res->path+1)) { // upload every changed file if (local_resource_is_changed(dir, db, local_res)) { printf("put: %s\n", local_res->path); DavResource *res = dav_resource_new(sn, local_res->path); if(sync_put_resource(dir, res, local_res)) { // TODO: I don't know what to do now } dav_resource_free(res); } // remove every locally available resource from db->resource // the remaining elements are all deleted files ucx_map_cstr_put(lclres, local_res->path, local_res); ucx_map_cstr_remove(db->resources, local_res->path); // TODO: element leaked } } ucx_list_free(resources); // delete all removed files UcxMapIterator i = ucx_map_iterator(db->resources); LocalResource *local; UCX_MAP_FOREACH(key, local, i) { if (!res_matches_filter(dir, local->path+1)) { if(sync_delete_remote_resource(sn, local)) { ucx_map_cstr_put(lclres, local->path, local); } } } ucx_map_free(db->resources); db->resources = lclres; // TODO: free res // store db if(store_db(db, dir->database)) { fprintf(stderr, "Cannot store sync db\n"); return -1; } return 0; } UcxList* local_scan(SyncDirectory *dir, SyncDatabase *db) { UcxList *resources = NULL; char *path = strdup("/"); UcxList *stack = ucx_list_prepend(NULL, path); while(stack) { // get a directory path from the stack and read all entries // if an entry is a directory, put it on the stack char *p = stack->data; stack = ucx_list_remove(stack, stack); char *local_path = util_concat_path(dir->path, p); DIR *local_dir = opendir(local_path); if(!local_dir) { fprintf(stderr, "Cannot open directory %s\n", local_path); } else { struct dirent *ent; while((ent = readdir(local_dir)) != NULL) { if(!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) { continue; } char *new_path = util_concat_path(p, ent->d_name); int isdir; LocalResource *res = local_resource_new(dir, db, new_path, &isdir); if(isdir) { stack = ucx_list_prepend(stack, new_path); } else if(res) { resources = ucx_list_append(resources, res); free(new_path); } else { free(new_path); } } closedir(local_dir); } free(local_path); free(p); } return resources; } UcxList* read_changes(SyncDirectory *dir, SyncDatabase *db) { UcxProperties *parser = ucx_properties_new(); parser->delimiter = ':'; UcxList *resources = NULL; sstr_t name; sstr_t value; char buf[STDIN_BUF_SIZE]; size_t r; while(!feof(stdin)) { r = fread(buf, 1, STDIN_BUF_SIZE, stdin); ucx_properties_fill(parser, buf, r); while(ucx_properties_next(parser, &name, &value)) { if(value.length == 0) { fprintf(stderr, "Wrong input\n"); continue; } if(value.ptr[0] == '"' && value.length > 2 && value.ptr[value.length - 1] == '"') { value.ptr[value.length - 1] = '\0'; value.ptr++; value.length -= 2; } value = sstrdup(value); if(!sstrcmp(name, S("put"))) { int isdir; LocalResource *res = local_resource_new(dir, db, value.ptr, &isdir); if(res) { resources = ucx_list_append(resources, res); } } else if(!sstrcmp(name, S("remove"))) { LocalResource *res = calloc(1, sizeof(LocalResource)); res->path = sstrdup(value).ptr; if(res) { ucx_map_sstr_put(db->remove, value, res); ucx_map_sstr_remove(db->resources, value); } } free(value.ptr); } } ucx_properties_free(parser); return resources; } LocalResource* local_resource_new(SyncDirectory *dir, SyncDatabase *db, char *path, int *isdir) { char *file_path = util_concat_path(dir->path, path); struct stat s; if(stat(file_path, &s)) { fprintf(stderr, "Cannot stat file %s\n", file_path); free(file_path); return NULL; } free(file_path); if(!S_ISDIR(s.st_mode)) { *isdir = 0; LocalResource *res = calloc(1, sizeof(LocalResource)); res->path = strdup(path); res->etag = NULL; res->last_modified = s.st_mtime; res->size = s.st_size; return res; } else { *isdir = 1; } return NULL; } int local_resource_is_changed(SyncDirectory *dir, SyncDatabase *db, LocalResource *res) { LocalResource *db_res = ucx_map_cstr_get(db->resources, res->path); if(db_res) { if(db_res->etag) { res->etag = strdup(db_res->etag); } if(db_res->last_modified == res->last_modified && db_res->size == res->size) { return 0; } } return 1; } int sync_put_resource(SyncDirectory *dir, DavResource *res, LocalResource *local) { char *local_path = util_concat_path(dir->path, res->path); FILE *in = fopen(local_path, "rb"); if(!in) { fprintf(stderr, "Cannot open file %s\n", local_path); free(local_path); return -1; } free(local_path); dav_set_content(res, in, (dav_read_func)fread); int ret = -1; for(;;) { if(dav_create(res)) { break; } if(dav_store(res)) { break; } ret = 0; break; } if(ret == 0) { // get new etag DavResource *up_res = dav_get(res->session, res->path, "D:getetag"); char *etag = dav_get_property(up_res, "D:getetag"); if(etag) { if(strlen(etag) > 2 && etag[0] == 'W' && etag[1] == '/') { etag = etag + 2; } } if(local->etag) { free(local->etag); } if(etag) { local->etag = strdup(etag); } else { local->etag = NULL; } dav_resource_free(up_res); } fclose(in); return ret; } int sync_delete_remote_resource(DavSession *sn, LocalResource *local_res) { DavResource *res = dav_get(sn, local_res->path, "D:getetag"); if(!res) { return sn->error == DAV_NOT_FOUND ? 0 : 1; } char *etag = dav_get_property(res, "D:getetag"); if(etag) { if(strlen(etag) > 2 && etag[0] == 'W' && etag[1] == '/') { etag = etag + 2; } } int ret = 0; if(etag && !strcmp(etag, local_res->etag)) { // local resource metadata == remote resource metadata // resource can be deleted printf("delete: %s\n", res->path); if(dav_delete(res)) { if(sn->error != DAV_NOT_FOUND) { fprintf(stderr, "Cannot delete resource %s\n", res->path); } } } else { ret = 1; } // cleanup dav_resource_free(res); return ret; }