dav/sync.c

/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 2016 Olaf Wintermann. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *   1. Redistributions of source code must retain the above copyright
 *      notice, this list of conditions and the following disclaimer.
 *
 *   2. Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in the
 *      documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <unistd.h>
#include <time.h>
#include <libxml/xmlerror.h>
#include <sys/types.h>
#include <ucx/string.h>
#include <ucx/utils.h>
#include <dirent.h>

#include <libidav/webdav.h>
#include <libidav/utils.h>

#include "config.h"
#include "scfg.h"
#include "sopt.h"
#include "db.h"
#include "error.h"
#include "assistant.h"

#include "sync.h"
#include "ucx/properties.h"

static DavContext *ctx;

static void xmlerrorfnc(void * c, const char * msg, ... ) {
    // nothing
}

int main(int argc, char **argv) {
    if(argc < 2) {
        fprintf(stderr, "Missing command\n");
        print_usage(argv[0]);
        return -1;
    }
    
    char *cmd = argv[1];
    CmdArgs *args = cmd_parse_args(argc - 2, argv + 2);
    if(!args) {
        print_usage(argv[0]);
        return -1;
    }
    int ret = EXIT_FAILURE;
    
    int loadcfg = 1;
    if(!strcasecmp(cmd, "version") || !strcasecmp(cmd, "-version") || !strcasecmp(cmd, "--version")) {
        fprintf(stderr, "dav-sync %s\n", DAV_VERSION);
        loadcfg = 0;
    }
    
    if(loadcfg) {
        xmlGenericErrorFunc fnc = xmlerrorfnc;
        initGenericErrorDefaultFunc(&fnc);
        ctx = dav_context_new();
        ctx->key_input = util_key_input;
        load_config(ctx);

        // copy proxy config
        memcpy(ctx->http_proxy, get_http_proxy(), sizeof(Proxy));
        memcpy(ctx->https_proxy, get_https_proxy(), sizeof(Proxy));
        
        if(load_sync_config()) {
            return EXIT_FAILURE;
        }
    }
    
    if(!strcmp(cmd, "pull")) {
        ret = cmd_pull(args);
    } else if(!strcmp(cmd, "push")) {
        ret = cmd_push(args);
    } else if(!strcmp(cmd, "resolve-conflicts")) {
        ret = cmd_resolve_conflicts(args);
    } else if(!strcmp(cmd, "delete-conflicts")) {
        ret = cmd_delete_conflicts(args);
    } else if(!strcmp(cmd, "trash-info")) {
        ret = cmd_trash_info(args);
    } else if(!strcmp(cmd, "empty-trash")) {
        ret = cmd_empty_trash(args);
    } else if(!strcmp(cmd, "add-directory")) {
        ret = cmd_add_directory(args);
    } else if(!strcmp(cmd, "list-directories")) {
        ret = list_syncdirs();
    }
    
    // TODO: cleanup sync config (don't forget to call regfree for regex)
    
    return ret;
}

void print_usage(char *cmd) {
    fprintf(stderr, "Usage: %s command [options] arguments...\n\n", cmd);
    
    fprintf(stderr, "Commands:\n");
    fprintf(stderr, "        pull [-c] <directory>\n");
    fprintf(stderr, "        push [-c] <directory>\n");
    fprintf(stderr, "        resolve-conflicts <directory>\n");
    fprintf(stderr, "        delete-conflicts <directory>\n");
    fprintf(stderr, "        trash-info <directory>\n");
    fprintf(stderr, "        empty-trash <directory>\n\n");
    
    fprintf(stderr, "Options:\n");
    fprintf(stderr, "        -c         Disable conflict detection\n");
    //fprintf(stderr, "        -r         Read changes from stdin\n\n");
    fprintf(stderr, "        -v         verbose output (all commands)\n\n");
    
    fprintf(stderr, "Config commands:\n");
    fprintf(stderr, "        add-directory\n");
    fprintf(stderr, "        list-directories\n\n");
}

static int res_matches_filter(SyncDirectory *dir, char *res_path) {
    // trash filter
    if (dir->trash) {
        sstr_t rpath = sstr(util_concat_path(dir->path, res_path));
        if (sstrprefix(rpath, sstr(dir->trash))) {
            free(rpath.ptr);
            return 1;
        }
        free(rpath.ptr);
    }
    
    // include/exclude filter
    UCX_FOREACH(inc, dir->include) {
        regex_t* pattern = (regex_t*) inc->data;
        if (regexec(pattern, res_path, 0, NULL, 0) == 0) {
            UCX_FOREACH(exc, dir->exclude) {
                regex_t* pattern = (regex_t*) exc->data;
                if (regexec(pattern, res_path, 0, NULL, 0) == 0) {
                    return 1;
                }
            }
            return 0;
        }
    }
    return 1;
}

static DavSession* create_session(DavContext *ctx, Repository *repo, char *url) {
    DavSession *sn = dav_session_new_auth(
            ctx,
            url,
            repo->user,
            repo->password);
    sn->flags = get_repository_flags(repo);
    sn->key = dav_context_get_key(ctx, repo->default_key);
    curl_easy_setopt(sn->handle, CURLOPT_HTTPAUTH, repo->authmethods);
    curl_easy_setopt(sn->handle, CURLOPT_SSLVERSION, repo->ssl_version);
    if(repo->cert) {
        curl_easy_setopt(sn->handle, CURLOPT_CAPATH, repo->cert);
    }
    if(!repo->verification) {
        curl_easy_setopt(sn->handle, CURLOPT_SSL_VERIFYPEER, 0);
        curl_easy_setopt(sn->handle, CURLOPT_SSL_VERIFYHOST, 0);
    }
    return sn;
}

int cmd_pull(CmdArgs *a) {
    if(a->argc != 1) {
        fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many");
        return -1;
    }
    
    SyncDirectory *dir = scfg_get_dir(a->argv[0]);
    if(!dir) {
        fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]);
        return -1;
    }
    
    Repository *repo = get_repository(sstr(dir->repository));
    if(!repo) {
        fprintf(stderr, "Unkown repository %s\n", dir->name);
        return -1;
    }
    
    SyncDatabase *db = load_db(dir->database);
    if(!db) {
        fprintf(stderr, "Cannot load database file: %s\n", dir->database);
        return -1;
    }
    remove_deleted_conflicts(dir, db);
    
    char *new_url = NULL;
    if(dir->collection) {
        new_url = util_concat_path(repo->url, dir->collection);
    }
    DavSession *sn = create_session(ctx, repo, new_url ? new_url : repo->url);
    if(new_url) {
        free(new_url);
    }
    if (cmd_getoption(a, "verbose")) {
        curl_easy_setopt(sn->handle, CURLOPT_VERBOSE, 1L);
        curl_easy_setopt(sn->handle, CURLOPT_STDERR, stderr);
    }
    
    DavResource *ls = dav_query(sn, "select D:getetag,idav:status from / with depth = infinity");
    if(!ls) {
        print_resource_error(sn, "/");
        fprintf(stderr, "Abort\n");
        
        dav_session_destroy(sn);
        // TODO: free
        return -1;
    }
    
    int sync_success = 0;
    int sync_delete = 0;
    int sync_error = 0;
    
    if(!ls->children) {
        // TODO: free
        fprintf(stderr, "Repository is empty\n");
        return 0; // empty repository
    }
    
    UcxMap *svrres = ucx_map_new(db->resources->count);
    
    UcxList *stack = ucx_list_prepend(NULL, ls->children);
    while(stack) {
        DavResource *res = stack->data;
        stack = ucx_list_remove(stack, stack);
         
        while(res) {
            if (res_matches_filter(dir, res->path)) {
                res = res->next;
                continue;
            }
            
            char *status = dav_get_property(res, "idav:status");
            if(status && !strcmp(status, "broken")) {
                res = res->next;
                continue;
            }
            
            // download the resource
            if(sync_get_resource(a, dir, res, db, &sync_success)) {
                fprintf(stderr, "sync_get_resource failed for resource: %s\n", res->path);
                sync_error++;
            }
            
            // add every resource from the server to svrres
            // then db-resources contains only resources which are not on the
            // server
            LocalResource *local = ucx_map_cstr_get(db->resources, res->path);
            ucx_map_cstr_put(svrres, res->path, local);
            ucx_map_cstr_remove(db->resources, res->path);
            
            if(res->children) {
                stack = ucx_list_prepend(stack, res->children);
            }
            res = res->next;
        }
    }
    
    // delete every remotely removed resource
    UcxMapIterator i = ucx_map_iterator(db->resources);
    LocalResource *local;
    UcxList *rmdirs = NULL;
    UCX_MAP_FOREACH(key, local, i) {
        if (res_matches_filter(dir, local->path)) {
            continue;
        }
        // sync_remove_resource does all necessary tests
        int ret = sync_remove_local_resource(dir, local);
        if(ret == -1) {
            rmdirs = ucx_list_append(rmdirs, local);
        } else if(ret == 0) {
            sync_delete++;
        }
    }
    UCX_FOREACH(elm, rmdirs) {
        LocalResource *local_dir = elm->data;
        sync_remove_local_directory(dir, local_dir);
    }
    ucx_map_free(db->resources);
    db->resources = svrres;
    
    // store db
    if(store_db(db, dir->database)) {
        fprintf(stderr, "Cannot store sync db\n");
        fprintf(stderr, "Abort\n");
        return -1; // TODO: don't return here
    }
    
    // TODO: cleanup - BUT DONT CLEANUP SYNC CONFIG (do this in main!)
    dav_session_destroy(sn);
    
    // Report
    char *str_success = sync_success == 1 ? "file" : "files";
    char *str_delete = sync_delete == 1 ? "file" : "files";
    char *str_error = sync_error == 1 ? "error" : "errors";
    printf("Result: %d %s pulled, %d %s deleted, %d %s\n",
            sync_success, str_success,
            sync_delete,str_delete,
            sync_error, str_error);
    
    return 0;
}

int sync_get_resource(
        CmdArgs *a,
        SyncDirectory *dir,
        DavResource *res,
        SyncDatabase *db,
        int *counter)
{
    int cdt = cmd_getoption(a, "conflict") ? 0 : 1; // conflict detection
    
    LocalResource *local = ucx_map_cstr_get(db->resources, res->path);
    char *local_path = util_concat_path(dir->path, res->path);
    
    char *etag = dav_get_property(res, "D:getetag");
    struct stat s;
    if(local && !res->iscollection) {
        int exists = 1;
        if(stat(local_path, &s)) {
            // Ignore the fact, that the file is locally removed. If the
            // server has an updated version, we read the file or the
            // next push will delete it on the server.
            if(errno != ENOENT) {
                fprintf(stderr, "Cannot stat file: %s\n", local_path);
                free(local_path);
                return -1;
            } else {
                exists = 0;
            }
        }
              
        if(local->etag) {
            sstr_t e = sstr(etag);
            if(sstrprefix(e, S("W/"))) {
                e = sstrsubs(e, 2);
            }
            if(!strcmp(e.ptr, local->etag)) {
                // resource is already up-to-date on the client
                return 0;
            }
        }
        
        if(cdt && exists && s.st_mtime != local->last_modified) {
            // file modified on the server and on the client
            rename_conflict_file(dir, db, local->path);
        }
    } else {
        if(stat(local_path, &s)) {
            if(errno != ENOENT) {
                fprintf(stderr, "Cannot stat file: %s\n", local_path);
            }
        } else if(S_ISDIR(s.st_mode)) {
            //fprintf(stderr, "Error: file %s is a directory\n", local_path);
        } else if(cdt) {
            // rename file on conflict
            rename_conflict_file(dir, db, res->path);
        }
    }
      
    int ret = 0;
    char *tmp_path = create_tmp_download_path(local_path);
    if(res->iscollection) {
        mode_t mode = S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH;
        if(util_mkdir(local_path, mode) && errno != EEXIST) {
            ret = -1;
        }
        
        if(ret == 0) {
            if(!local) {
                // new local resource
                local = calloc(1, sizeof(LocalResource));
                local->path = util_concat_path(res->path, "/");
                local->last_modified = s.st_mtime;
                ucx_map_cstr_put(db->resources, local->path, local);
            }
        }
    } else {
        if(!tmp_path) {
            fprintf(stderr, "Cannot create tmp path for %s\n", local_path);
            free(local_path);
            return -1;
        }
        FILE *out = fopen(tmp_path, "wb");
        if(!out) {
            fprintf(stderr, "Cannot open output file: %s\n", local_path);
            free(local_path);
            free(tmp_path);
            return -1;
        }
        printf("get: %s\n", res->path);
        if(dav_get_content(res, out, (dav_write_func)fwrite)) {
            ret = -1;
        }
        fclose(out);
        
        if(ret == 0) {
            (*counter)++;
            
            if(dir->trash && dir->backuppull) {
                move_to_trash(dir, local_path);
            }
            if(rename(tmp_path, local_path)) {
                fprintf(
                        stderr,
                        "Cannot rename file %s to %s\n",
                        tmp_path,
                        local_path);
                perror("");
                free(tmp_path);
                free(local_path);
                return -1;
            }
            
            if(stat(local_path, &s)) {
                fprintf(stderr, "Cannot stat file: %s\n", local_path);
                perror("");
            }
            
            if(!local) {
                // new local resource
                local = calloc(1, sizeof(LocalResource));
                local->path = strdup(res->path);
                ucx_map_cstr_put(db->resources, local->path, local);
            }
            
            if(local->etag) {
                free(local->etag);
            }
            // set metadata from stat
            local->etag = etag;
            local->last_modified = s.st_mtime;
            local->size = s.st_size;
        } else {
            if(unlink(tmp_path)) {
                fprintf(stderr, "Cannot remove tmp file: %s\n", tmp_path);
            }
        }
    }
    
    free(tmp_path);
    free(local_path);
    return ret;
}

int sync_remove_local_resource(SyncDirectory *dir, LocalResource *res) {
    char *local_path = util_concat_path(dir->path, res->path);
    struct stat s;
    if(stat(local_path, &s)) {
        free(local_path);
        return -2;
    }
    
    if(S_ISDIR(s.st_mode)) {
        free(local_path);
        return -1;
    }
    
    if(s.st_mtime != res->last_modified) {
        free(local_path);
        return -2;
    }
    
    printf("delete: %s\n", res->path);
    int ret = 0;
    if(dir->trash) {
        move_to_trash(dir, local_path);
    } else if(unlink(local_path)) {
        fprintf(stderr, "Cannot remove file %s\n", local_path);
        ret = -2;
    }
    free(local_path);
    
    return ret;
}

void sync_remove_local_directory(SyncDirectory *dir, LocalResource *res) {
    char *local_path = util_concat_path(dir->path, res->path);
    
    printf("delete: %s\n", res->path);
    if(rmdir(local_path)) {
        // TODO
    }
    
    free(local_path);
}

void rename_conflict_file(SyncDirectory *dir, SyncDatabase *db, char *path) {
    char *local_path = util_concat_path(dir->path, path);
    char *parent = util_parent_path(local_path);
    
    int rev = 0;
    struct stat s;
    int loop = 1;
    do {
        char *res_parent = util_parent_path(path);
        char *res_name = util_resource_name(path);
        
        sstr_t new_path = ucx_sprintf(
            "%sorig.%d.%s",
            parent,
            rev,
            res_name);
        sstr_t new_res_path = ucx_sprintf(
            "%sorig.%d.%s",
            res_parent,
            rev,
            res_name);
        
        
        if(stat(new_path.ptr, &s)) {
            if(errno == ENOENT) {
                loop = 0;
                printf("conflict: %s\n", local_path);
                if(rename(local_path, new_path.ptr)) {
                    //printf("errno: %d\n", errno);
                    fprintf(
                            stderr,
                            "Cannot rename file %s to %s\n",
                            local_path,
                            new_path.ptr);
                } else {
                    LocalResource *conflict = calloc(1, sizeof(LocalResource));
                    conflict->path = strdup(new_res_path.ptr);
                    ucx_map_cstr_put(db->conflict, new_res_path.ptr, conflict);
                }
            }
        }
        rev++;
        free(res_parent);
        free(new_path.ptr);
        free(new_res_path.ptr);
        
    } while(loop);
    free(parent);
    free(local_path);
}

char* create_tmp_download_path(char *path) {
    char *new_path = NULL;
    char *parent = util_parent_path(path);
    for (int i=0;;i++) {
        sstr_t np = ucx_asprintf(
        ucx_default_allocator(),
            "%sdownload%d-%s",
            parent,
            i,
            util_resource_name(path));
        
        struct stat s;
        if(stat(np.ptr, &s)) {
            if(errno == ENOENT) {
                new_path = np.ptr;
            }
            break;
        }
        free(np.ptr);
    };
    
    free(parent);
    return new_path;
}

void move_to_trash(SyncDirectory *dir, char *path) {
    char *new_path = NULL;
    for (int i=0;;i++) {
        sstr_t np = ucx_asprintf(
        ucx_default_allocator(),
            "%s%d-%s",
            dir->trash,
            i,
            util_resource_name(path));
        
        struct stat s;
        if(stat(np.ptr, &s)) {
            if(errno == ENOENT) {
                new_path = np.ptr;
            }
            break;
        }
        free(np.ptr);
    };
    
    if(!new_path) {
        fprintf(stderr, "Cannot move file %s to trash.\n", path);
        return;
    }
    
    if(rename(path, new_path)) {
        //printf("errno: %d\n", errno);
        fprintf(
                stderr,
                "Cannot rename file %s to %s\n",
                path,
                new_path);
    }
    
    free(new_path);
}

static int res_isconflict(SyncDatabase *db, LocalResource *res) {
    return ucx_map_cstr_get(db->conflict, res->path) ? 1 : 0;
}

int cmd_push(CmdArgs *a) {
    if(a->argc != 1) {
        fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many");
        return -1;
    }
    
    SyncDirectory *dir = scfg_get_dir(a->argv[0]);
    if(!dir) {
        fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]);
        return -1;
    }
    
    Repository *repo = get_repository(sstr(dir->repository));
    if(!repo) {
        fprintf(stderr, "Unkown repository %s\n", dir->name);
        return -1;
    }
    
    SyncDatabase *db = load_db(dir->database);
    if(!db) {
        fprintf(stderr, "Cannot load database file: %s\n", dir->database);
        return -1;
    }
    remove_deleted_conflicts(dir, db);
    
    char *new_url = NULL;
    if(dir->collection) {
        new_url = util_concat_path(repo->url, dir->collection);
    }
    DavSession *sn = create_session(ctx, repo, new_url ? new_url : repo->url);
    if(new_url) {
        free(new_url);
    }
    if (cmd_getoption(a, "verbose")) {
        curl_easy_setopt(sn->handle, CURLOPT_VERBOSE, 1L);
        curl_easy_setopt(sn->handle, CURLOPT_STDERR, stderr);
    }
    
    DavResource *testsvr = dav_query(sn, "select - from / with depth = 0");
    if(!testsvr) {
        print_resource_error(sn, "/");
        dav_session_destroy(sn);
        fprintf(stderr, "Abort\n");
        return -1;
    }
    
    int cdt = cmd_getoption(a, "conflict") ? 0 : 1; // conflict detection
    
    int sync_success = 0;
    int sync_delete = 0;
    int sync_skipped = 0;
    int sync_error = 0;
    
    // upload all changed files
    //UcxList *resources = cmd_getoption(a, "read") ?
    //        read_changes(dir, db) : local_scan(dir, db);
    UcxList *resources = local_scan(dir, db);
    
    UcxMap *lclres = ucx_map_new(db->resources->count);
    int ret = 0;
    UCX_FOREACH(elm, resources) {
        LocalResource *local_res = elm->data;
        if (!res_matches_filter(dir, local_res->path+1)) {
            if(res_isconflict(db, local_res)) {
                printf("skip: %s\n", local_res->path);
                sync_skipped++;
                continue;
            }
            
            // upload every changed file
            if (local_resource_is_changed(dir, db, local_res)) {
                DavResource *res = dav_resource_new(sn, local_res->path);
                if(!res) {
                    print_resource_error(sn, local_res->path);
                    ret = -1;
                    sync_error++;
                }
                
                if(local_res->isdirectory) {
                    printf("mkcol: %s\n", local_res->path);
                    if(sync_mkdir(dir, res, local_res) && sn->error != DAV_METHOD_NOT_ALLOWED) {
                        print_resource_error(sn, res->path);
                        ret = -1;
                        sync_error++;
                    }
                } else {
                    if(cdt && remote_resource_is_changed(sn, dir, db, local_res)) {
                        printf("conflict: %s\n", local_res->path);
                        sync_skipped++;
                        continue;
                    }
                    
                    printf("put: %s\n", local_res->path);
                    if(sync_put_resource(dir, res, local_res, &sync_success)) {
                        sync_error++;
                        print_resource_error(sn, res->path);
                        ret = -1;
                        sync_error++;
                    }
                }
                dav_resource_free(res);
            }
            
            // remove every locally available resource from db->resource
            // the remaining elements are all deleted files
            ucx_map_cstr_put(lclres, local_res->path, local_res);
            ucx_map_cstr_remove(db->resources, local_res->path); // TODO: element leaked
        }
    }
    ucx_list_free(resources);
    
    // delete all removed files
    if(ret == 0) {
        UcxMapIterator i = ucx_map_iterator(db->resources);
        LocalResource *local;
        UCX_MAP_FOREACH(key, local, i) {
            if (!res_matches_filter(dir, local->path+1)) {
                if(sync_delete_remote_resource(sn, local, &sync_delete)) {
                    ucx_map_cstr_put(lclres, local->path, local);
                    if(sn->error != DAV_NOT_FOUND) {
                        print_resource_error(sn, local->path);
                        sync_error++;
                        break;
                    }
                }
            }
        }
    }
    ucx_map_free(db->resources);
    db->resources = lclres;
    
    // store db
    if(store_db(db, dir->database)) {
        fprintf(stderr, "Cannot store sync db\n");
        return -1;
    }
    
    // TODO: free res
    
    // Report
    char *str_success = sync_success == 1 ? "file" : "files";
    char *str_delete = sync_delete == 1 ? "file" : "files";
    char *str_skipped = sync_delete == 1 ? "file" : "files";
    char *str_error = sync_error == 1 ? "error" : "errors";
    printf("Result: %d %s pushed, %d %s deleted, %d %s skipped, %d %s\n",
            sync_success, str_success,
            sync_delete,str_delete,
            sync_skipped,str_skipped,
            sync_error, str_error);
    
    return 0;
}

UcxList* local_scan(SyncDirectory *dir, SyncDatabase *db) {
    UcxList *resources = NULL;
    
    char *path = strdup("/");
    UcxList *stack = ucx_list_prepend(NULL, path);
    while(stack) {
        // get a directory path from the stack and read all entries
        // if an entry is a directory, put it on the stack
        
        char *p = stack->data;
        stack = ucx_list_remove(stack, stack);
        char *local_path = util_concat_path(dir->path, p);
        DIR *local_dir = opendir(local_path);
        
        if(!local_dir) {
            fprintf(stderr, "Cannot open directory %s\n", local_path);
        } else {
            struct dirent *ent;
            while((ent = readdir(local_dir)) != NULL) {
                if(!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) {
                    continue;
                }
                
                char *new_path = util_concat_path(p, ent->d_name);
                int isdir;
                LocalResource *res = local_resource_new(dir, db, new_path, &isdir);
                if(isdir) {
                    resources = ucx_list_append(resources, res);
                    stack = ucx_list_prepend(stack, new_path);
                } else if(res) {
                    resources = ucx_list_append(resources, res);
                    free(new_path);
                } else {
                    free(new_path);
                }
            }
            closedir(local_dir);
 
        }
        free(local_path);
        free(p);
    }
    
    return resources;
}

UcxList* read_changes(SyncDirectory *dir, SyncDatabase *db) {
    UcxProperties *parser = ucx_properties_new();
    parser->delimiter = ':';
    
    UcxList *resources = NULL;
    sstr_t name;
    sstr_t value;
    
    char buf[STDIN_BUF_SIZE];
    size_t r;
    while(!feof(stdin)) {
        r = fread(buf, 1, STDIN_BUF_SIZE, stdin);
        ucx_properties_fill(parser, buf, r);
        while(ucx_properties_next(parser, &name, &value)) {
            if(value.length == 0) {
                fprintf(stderr, "Wrong input\n");
                continue;
            }
            if(value.ptr[0] == '"'
                    && value.length > 2
                    && value.ptr[value.length - 1] == '"')
            {
                value.ptr[value.length - 1] = '\0';
                value.ptr++;
                value.length -= 2;
            }
            value = sstrdup(value);
            
            if(!sstrcmp(name, S("put"))) {
                int isdir;
                LocalResource *res = local_resource_new(dir, db, value.ptr, &isdir);
                if(res) {
                    resources = ucx_list_append(resources, res);
                }
            } else if(!sstrcmp(name, S("remove"))) {
                LocalResource *res = calloc(1, sizeof(LocalResource));
                res->path = sstrdup(value).ptr;
                if(res) {
                    //ucx_map_sstr_put(db->remove, value, res);
                    ucx_map_sstr_remove(db->resources, value);
                }
                
            }
            
            free(value.ptr);
        }
    }
    ucx_properties_free(parser);    
    
    return resources;
}

LocalResource* local_resource_new(SyncDirectory *dir, SyncDatabase *db, char *path, int *isdir) {
    char *file_path = util_concat_path(dir->path, path);
    struct stat s;
    if(stat(file_path, &s)) {
        fprintf(stderr, "Cannot stat file %s\n", file_path);
        free(file_path);
        return NULL;
    }
    free(file_path);

    if(!S_ISDIR(s.st_mode)) {
        *isdir = 0;
        LocalResource *res = calloc(1, sizeof(LocalResource));
        res->path = strdup(path);
        res->etag = NULL;
        res->last_modified = s.st_mtime;
        res->size = s.st_size;
        return res;
    } else {
        *isdir = 1;
        LocalResource *res = calloc(1, sizeof(LocalResource));
        res->path = util_concat_path(path, "/");
        res->last_modified = s.st_mtime;
        res->isdirectory = 1;
        return res;
    }
    return NULL;
}

int local_resource_is_changed(SyncDirectory *dir, SyncDatabase *db, LocalResource *res) {
    LocalResource *db_res = ucx_map_cstr_get(db->resources, res->path);
    if(db_res) {
        if(db_res->etag) {
            res->etag = strdup(db_res->etag);
        }
        
        if(db_res->last_modified == res->last_modified && db_res->size == res->size) {
            return 0;
        }
    }
    return 1;
}

int remote_resource_is_changed(
        DavSession *sn,
        SyncDirectory *dir,
        SyncDatabase *db,
        LocalResource *res)
{
    DavResource *remote = dav_get(sn, res->path, "D:getetag");
    int ret = 0;
    if(remote) {
        char *etag = dav_get_property(remote, "D:getetag");
        if(etag && res->etag) {
            sstr_t e = sstr(etag);
            if(sstrprefix(e, S("W/"))) {
                e = sstrsubs(e, 2);
            }
            if(strcmp(e.ptr, res->etag)) {
                ret = 1;
            }
        }
        dav_resource_free(remote);
    }
    return ret;
}


int sync_set_status(DavResource *res, char *status) {
    DavResource *resource = dav_resource_new(res->session, res->path);
    dav_set_property(resource, "idav:status", status);
    int ret = dav_store(resource);
    dav_resource_free(resource);
    return ret;
}

int sync_remove_status(DavResource *res) {
    DavResource *resource = dav_resource_new(res->session, res->path);
    dav_remove_property(resource, "idav:status");
    int ret = dav_store(resource);
    dav_resource_free(resource);
    return ret;
}

int sync_put_resource(
        SyncDirectory *dir,
        DavResource *res,
        LocalResource *local,
        int *counter)
{
    char *local_path = util_concat_path(dir->path, res->path);
    
    struct stat s;
    if(stat(local_path, &s)) {
        fprintf(stderr, "cannot stat file: %s\n", local_path);
        perror("");
        free(local_path);
        return -1;
    }
    
    FILE *in = fopen(local_path, "rb");
    if(!in) {
        fprintf(stderr, "Cannot open file %s\n", local_path);
        free(local_path);
        return -1;
    }
    
    dav_set_content(res, in, (dav_read_func)fread);
    
    int ret = -1;
    int created = 0;
    for(int i=0;i<dir->max_retry;i++) {
        if(!created && dav_create(res)) {
            continue;
        }
        created = 1;
        if(dav_store(res)) {
            continue;
        }
        ret = 0;
        break;
    }
    

    if(ret == 0) {
        (*counter)++;
        
        // check contentlength and get new etag
        DavResource *up_res = dav_get(res->session, res->path, "D:getetag");
        
        if(up_res) {
            // the new content length must be equal or greater than the file size
            if(up_res->contentlength < s.st_size) {
                fprintf(stderr, "Incomplete Upload: %s", local_path);
                ret = -1;
                // try to set the resource status to 'broken'
                sync_set_status(res, "broken");
            } else {
                // everything seems fine, we can update the local resource
                char *etag = dav_get_property(up_res, "D:getetag");
                if(etag) {
                    if(strlen(etag) > 2 && etag[0] == 'W' && etag[1] == '/') {
                        etag = etag + 2;
                    } 
                }

                if(local->etag) {
                    free(local->etag);
                }

                if(etag) {
                    local->etag = strdup(etag);
                } else {
                    local->etag = NULL;
                }
                dav_resource_free(up_res);
                
                sync_remove_status(res);
            }
        }
    } else {
        ret = -1;
        sync_set_status(res, "broken");
    }
    
    fclose(in);
    free(local_path);
    
    return ret;
}

int sync_mkdir(SyncDirectory *dir, DavResource *res, LocalResource *local) {
    res->iscollection = 1;
    int ret = -1;
    for(int i=0;i<dir->max_retry;i++) {
        if(dav_create(res)) {
            continue;
        }
        ret = 0;
        break;
    }
    return ret;
}

int sync_delete_remote_resource(
        DavSession *sn,
        LocalResource *local_res,
        int *counter)
{
    DavResource *res = dav_get(sn, local_res->path, "D:getetag");
    if(!res) {
        return sn->error == DAV_NOT_FOUND ? 0 : 1;
    }
    
    int ret = 0;
    if(res->iscollection) {
        printf("delete: %s\n", res->path);
        if(dav_delete(res)) {
            ret = 1;
            fprintf(stderr, "Cannot delete resource %s\n", res->path);
        }
    } else {
        char *etag = dav_get_property(res, "D:getetag");
        if(etag) {
            if(strlen(etag) > 2 && etag[0] == 'W' && etag[1] == '/') {
                etag = etag + 2;
            } 
        }

        if(etag && !strcmp(etag, local_res->etag)) {
            // local resource metadata == remote resource metadata
            // resource can be deleted
            printf("delete: %s\n", res->path);
            if(dav_delete(res)) {
                if(sn->error != DAV_NOT_FOUND) {
                    fprintf(stderr, "Cannot delete resource %s\n", res->path);
                    ret = 1;
                }
            } else {
                (*counter)++;
            }
        }
    }
    
    // cleanup
    dav_resource_free(res);
    
    return ret;
}


void remove_deleted_conflicts(SyncDirectory *dir, SyncDatabase *db) {
    char **dc = calloc(sizeof(void*), db->conflict->count);
    int numdc = 0;
    
    UcxMapIterator i = ucx_map_iterator(db->conflict);
    LocalResource *res;
    UCX_MAP_FOREACH(key, res, i) {
        char *path = util_concat_path(dir->path, res->path);
        struct stat s;
        if(stat(path, &s)) {
            if(errno == ENOENT) {
                dc[numdc] = res->path;
                numdc++;
            } else {
                fprintf(stderr, "Cannot stat file: %s\n", path);
                perror("");
            }
        }
        free(path);
    }
    
    for(int i=0;i<numdc;i++) {
        ucx_map_cstr_remove(db->conflict, dc[i]);
    }
}

int cmd_resolve_conflicts(CmdArgs *a) {
    if(a->argc != 1) {
        fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many");
        return -1;
    }
    
    SyncDirectory *dir = scfg_get_dir(a->argv[0]);
    if(!dir) {
        fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]);
        return -1;
    }
    
    SyncDatabase *db = load_db(dir->database);
    if(!db) {
        fprintf(stderr, "Cannot load database file: %s\n", dir->database);
        return -1;
    }
    
    // remove conflicts
    int num_conflict = db->conflict->count;
    //TODO: ucx_map_free_content(db->conflict, destr);
    ucx_map_clear(db->conflict);
    
    // store db
    if(store_db(db, dir->database)) {
        fprintf(stderr, "Cannot store sync db\n");
        fprintf(stderr, "Abort\n");
        return -1; // TODO: don't return here
    }
    
    // Report
    char *str_conflict = num_conflict == 1 ? "conflict" : "conflicts";
    printf("Result: %d %s resolved\n", num_conflict, str_conflict);
    
    return 0;
}

int cmd_delete_conflicts(CmdArgs *a) {
    if(a->argc != 1) {
        fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many");
        return -1;
    }
    
    SyncDirectory *dir = scfg_get_dir(a->argv[0]);
    if(!dir) {
        fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]);
        return -1;
    }
    
    SyncDatabase *db = load_db(dir->database);
    if(!db) {
        fprintf(stderr, "Cannot load database file: %s\n", dir->database);
        return -1;
    }
    
    int num_del = 0;
    int num_err = 0;
    
    // delete all conflict files
    UcxMapIterator i = ucx_map_iterator(db->conflict);
    LocalResource *res;
    UCX_MAP_FOREACH(key, res, i) {
        printf("delete: %s\n", res->path);
        char *path = util_concat_path(dir->path, res->path);
        if(unlink(path)) {
            if(errno != ENOENT) {
                perror("unlink");
                num_err++;
            }
        } else {
            num_del++;
        }
        free(path);
    }
    //TODO: ucx_map_free_content(db->conflict, destr);
    ucx_map_clear(db->conflict);
    
    // store db
    if(store_db(db, dir->database)) {
        fprintf(stderr, "Cannot store sync db\n");
        fprintf(stderr, "Abort\n");
        return -1; // TODO: don't return here
    }
    
    // Report
    char *str_delete = num_del == 1 ? "file" : "files";
    char *str_error = num_err == 1 ? "error" : "errors";
    printf("Result: %d conflict %s deleted, %d %s\n",
            num_del, str_delete,
            num_err, str_error);
    
    return 0;
}


// TODO: remove code dup (main.c ls_size_str)
static char* size_str(uint64_t size) {
    char *str = malloc(16);
    
    if(size < 0x400) {
        snprintf(str, 16, "%" PRIu64 " bytes", size);
    } else if(size < 0x100000) {
        float s = (float)size/0x400;
        int diff = (s*100 - (int)s*100);
        if(diff > 90) {
            diff = 0;
            s += 0.10f;
        }
        if(size < 0x2800 && diff != 0) {
            // size < 10 KiB
            snprintf(str, 16, "%.1f KiB", s);
        } else {
            snprintf(str, 16, "%.0f KiB", s);
        }
    } else if(size < 0x40000000) {
        float s = (float)size/0x100000;
        int diff = (s*100 - (int)s*100);
        if(diff > 90) {
            diff = 0;
            s += 0.10f;
        }
        if(size < 0xa00000 && diff != 0) {
            // size < 10 MiB
            snprintf(str, 16, "%.1f MiB", s);
        } else {
            size /= 0x100000;
            snprintf(str, 16, "%.0f MiB", s);
        }
    } else if(size < 0x1000000000ULL) {
        float s = (float)size/0x40000000;
        int diff = (s*100 - (int)s*100);
        if(diff > 90) {
            diff = 0;
            s += 0.10f;
        }
        if(size < 0x280000000 && diff != 0) {
            // size < 10 GiB
            snprintf(str, 16, "%.1f GiB", s);
        } else {
            size /= 0x40000000;
            snprintf(str, 16, "%.0f GiB", s);
        }
    } else {
        size /= 1024;
        float s = (float)size/0x40000000;
        int diff = (s*100 - (int)s*100);
        if(diff > 90) {
            diff = 0;
            s += 0.10f;
        }
        if(size < 0x280000000 && diff != 0) {
            // size < 10 TiB
            snprintf(str, 16, "%.1f TiB", s);
        } else {
            size /= 0x40000000;
            snprintf(str, 16, "%.0f TiB", s);
        }
    }
    return str;
}

int cmd_trash_info(CmdArgs *a) {
    if(a->argc != 1) {
        fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many");
        return -1;
    }
    
    SyncDirectory *syncdir = scfg_get_dir(a->argv[0]);
    if(!syncdir) {
        fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]);
        return -1;
    }
    
    if(!syncdir->trash) {
        printf("trash not configured for %s\n", syncdir->name);
        return 0;
    }
    
    DIR *dir = opendir(syncdir->trash);
    if(!dir) {
        fprintf(stderr, "cannot open trash directory: %s\n", syncdir->trash);
        perror("opendir");
        return -1;
    }
    
    uint64_t trashsize = 0;
    int count = 0;
    struct dirent *ent;
    while((ent = readdir(dir)) != NULL) {
        if(!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) {
            continue;
        }
        
        char *path = util_concat_path(syncdir->trash, ent->d_name);
        
        struct stat s;
        if(stat(path, &s)) {
            perror("stat");
        } else {
            trashsize += s.st_size;
        }
        count++;
        
        free(path);
    }
    closedir(dir);
    
    printf("path: %s\n", syncdir->trash);
    printf("%d %s\n", count, count == 1 ? "file" : "files");
    char *sizestr = size_str(trashsize);
    printf("%s\n", sizestr);
    free(sizestr);
    
    return 0;
}


int cmd_empty_trash(CmdArgs *a) {
    if(a->argc != 1) {
        fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many");
        return -1;
    }
    
    SyncDirectory *syncdir = scfg_get_dir(a->argv[0]);
    if(!syncdir) {
        fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]);
        return -1;
    }
    
    if(!syncdir->trash) {
        fprintf(stderr, "trash not configured for %s\n", syncdir->name);
        return -1;
    }
    
    DIR *dir = opendir(syncdir->trash);
    if(!dir) {
        fprintf(stderr, "cannot open trash directory: %s\n", syncdir->trash);
        perror("opendir");
        return -1;
    }
    
    struct dirent *ent;
    while((ent = readdir(dir)) != NULL) {
        if(!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) {
            continue;
        }
        
        char *path = util_concat_path(syncdir->trash, ent->d_name);
        printf("delete: %s\n", path);
        
        struct stat s;
        if(stat(path, &s)) {
            perror("stat");
            free(path);
            continue;
        }
        if(S_ISDIR(s.st_mode)) {
            if(rmdir(path)) {
                perror("unlink");
            }
        } else {
            if(unlink(path)) {
                perror("unlink");
            }
        }
        
        free(path);
    }
    closedir(dir);
    
    return 0;
}

int cmd_add_directory(CmdArgs *args) {
    printf("Each sync directory must have an unique name.\n");
    char *name = assistant_getcfg("name");
    if(!name) {
        fprintf(stderr, "Abort\n");
        return -1;
    }
    if(scfg_get_dir(name)) {
        fprintf(stderr, "Directory %s already exists.\nAbort\n", name);
        return -1;
    }
    
    printf("Enter local directory path.\n");
    char *path = assistant_getcfg("path");
    if(!path) {
        fprintf(stderr, "Abort\n");
        return -1;
    }
    
    printf("Specify webdav repository.\n");
    UcxList *repos = get_repositories();
    int i = 0;
    UCX_FOREACH(elm, repos) {
        Repository *r = elm->data;
        printf("%d) %s\n", i, r->name);
        i++;
    }
    char *repository = assistant_getcfg("repository");
    char *reponame = NULL;
    if(!repository) {
        fprintf(stderr, "Abort\n");
        return -1;
    }
    int64_t reponum = 0;
    if(util_strtoint(repository, &reponum)) {
        if(reponum < 0) {
            fprintf(stderr, "Wrong input.\nAbort\n");
            return -1;
        }
        UcxList *elm = ucx_list_get(repos, reponum);
        if(elm) {
            Repository *r = elm->data;
            reponame = r->name;
        } else {
            fprintf(stderr, "Wrong input.\nAbort\n");
            return -1;
        }
    } else {
        if(get_repository(sstr(repository))) {
            reponame = repository;
        } else {
            fprintf(stderr, "Repository %s doesn't exist.\nAbort\n", repository);
            return -1;
        }
    }
    
    printf("Enter collection relative to the repository base url.\n");
    char *collection = assistant_getdefcfg("collection", "/");
    
    char *db = generate_db_name(name);
    
    SyncDirectory dir;
    memset(&dir, 0, sizeof(SyncDirectory));
    dir.name = name;
    dir.path = path;
    dir.repository = reponame;
    dir.collection = collection;
    dir.trash = ".trash";
    dir.database = db;
    
    int ret = 0;
    if(add_directory(&dir)) {
        fprintf(stderr, "Cannot write sync.xml\n");
        ret = -1;
    } else {
        printf("\nAdded directory: %s (%s)\n", name, path);
    }
    
    free(name);
    free(path);
    free(repository);
    free(collection);
    free(db);
    
    return ret;
}