dav/sync.c

/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 2013 Olaf Wintermann. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *   1. Redistributions of source code must retain the above copyright
 *      notice, this list of conditions and the following disclaimer.
 *
 *   2. Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in the
 *      documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <unistd.h>
#include <time.h>
#include <libxml/xmlerror.h>
#include <sys/types.h>
#include <ucx/string.h>
#include <ucx/utils.h>
#include <dirent.h>

#include <libidav/webdav.h>
#include <libidav/utils.h>

#include "config.h"
#include "scfg.h"
#include "sopt.h"
#include "db.h"

#include "sync.h"
#include "ucx/properties.h"

static DavContext *ctx;

static void xmlerrorfnc(void * c, const char * msg, ... ) {
    // nothing
}

int main(int argc, char **argv) {
    if(argc < 2) {
        fprintf(stderr, "Missing command\n");
        print_usage(argv[0]);
        return -1;
    }
    
    char *cmd = argv[1];
    CmdArgs *args = cmd_parse_args(argc - 2, argv + 2);
    if(!args) {
        print_usage(argv[0]);
        return -1;
    }
    
    xmlGenericErrorFunc fnc = xmlerrorfnc;
    initGenericErrorDefaultFunc(&fnc);
    ctx = dav_context_new();
    load_config(ctx);
    
    // copy proxy config
    memcpy(ctx->http_proxy, get_http_proxy(), sizeof(Proxy));
    memcpy(ctx->https_proxy, get_https_proxy(), sizeof(Proxy));
    
    if(load_sync_config()) {
        return EXIT_FAILURE;
    }
    
    int ret = EXIT_FAILURE;
    if(!strcmp(cmd, "pull")) {
        ret = cmd_pull(args);
    } else if(!strcmp(cmd, "push")) {
        ret = cmd_push(args);
    }
    
    // TODO: cleanup sync config (don't forget to call regfree for regex)
    
    return ret;
}

void print_usage(char *cmd) {
    fprintf(stderr, "Usage: %s command [options] arguments...\n\n", cmd);
    
    fprintf(stderr, "Commands:\n");
    fprintf(stderr, "        pull [-c] <directory>\n");
    fprintf(stderr, "        push [-r] <directory>\n\n");
    
    fprintf(stderr, "Options:\n");
    fprintf(stderr, "        -c         Disable conflict detection\n");
    fprintf(stderr, "        -r         Read changes from stdin\n\n");
}

static int res_matches_filter(SyncDirectory *dir, char *res_path) {
    // trash filter
    if (dir->trash) {
        sstr_t rpath = sstr(util_concat_path(dir->path, res_path));
        if (sstrprefix(rpath, sstr(dir->trash))) {
            free(rpath.ptr);
            return 1;
        }
        free(rpath.ptr);
    }
    
    // include/exclude filter
    UCX_FOREACH(inc, dir->include) {
        regex_t* pattern = (regex_t*) inc->data;
        if (regexec(pattern, res_path, 0, NULL, 0) == 0) {
            UCX_FOREACH(exc, dir->exclude) {
                regex_t* pattern = (regex_t*) exc->data;
                if (regexec(pattern, res_path, 0, NULL, 0) == 0) {
                    return 1;
                }
            }
            return 0;
        }
    }
    return 1;
}

static DavSession* create_session(DavContext *ctx, Repository *repo, char *url) {
    DavSession *sn = dav_session_new_auth(
            ctx,
            url,
            repo->user,
            repo->password);
    curl_easy_setopt(sn->handle, CURLOPT_SSLVERSION, repo->ssl_version);
    sn->flags = get_repository_flags(repo);
    sn->key = dav_context_get_key(ctx, repo->default_key);
    return sn;
}

int cmd_pull(CmdArgs *a) {
    if(a->argc != 1) {
        fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many");
        return -1;
    }
    
    SyncDirectory *dir = scfg_get_dir(a->argv[0]);
    if(!dir) {
        fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]);
        return -1;
    }
    
    Repository *repo = get_repository(sstr(dir->repository));
    if(!repo) {
        fprintf(stderr, "Unkown repository %s\n", dir->name);
        return -1;
    }
    
    SyncDatabase *db = load_db(dir->database);
    if(!db) {
        fprintf(stderr, "Cannot load database file: %s\n", dir->database);
        return -1;
    }
    
    char *new_url = NULL;
    if(dir->collection) {
        new_url = util_concat_path(repo->url, dir->collection);
    }
    DavSession *sn = create_session(ctx, repo, new_url ? new_url : repo->url);
    if(new_url) {
        free(new_url);
    }
    if (cmd_getoption(a, "verbose")) {
        curl_easy_setopt(sn->handle, CURLOPT_VERBOSE, 1L);
        curl_easy_setopt(sn->handle, CURLOPT_STDERR, stderr);
    }
    
    DavResource *ls = dav_query(sn, "get D:getetag from / where lastmodified > 0 with depth -1");
    if(!ls) {
        fprintf(stderr, "Error\n");
        // TODO: free
        return -1;
    }
    
    if(!ls->children) {
        // TODO: free
        return 0; // empty repository
    }
    
    UcxMap *svrres = ucx_map_new(db->resources->count);
    
    UcxList *stack = ucx_list_prepend(NULL, ls->children);
    while(stack) {
        DavResource *res = stack->data;
        stack = ucx_list_remove(stack, stack);
         
        while(res) {
            if (res_matches_filter(dir, res->path)) {
                res = res->next;
                continue;
            }
            
            // download the resource
            if(sync_get_resource(a, dir, res, db)) {
                fprintf(stderr, "sync_get_resource failed for resource: %s\n", res->path);
            }
            
            // add every resource from the server to svrres
            // then db-resources only contains resources which are not on the
            // server
            LocalResource *local = ucx_map_cstr_get(db->resources, res->path);
            ucx_map_cstr_put(svrres, res->path, local);
            ucx_map_cstr_remove(db->resources, res->path);
            
            if(res->children) {
                stack = ucx_list_prepend(stack, res->children);
            }
            res = res->next;
        }
    }
    
    // delete every remotely removed resource
    UcxMapIterator i = ucx_map_iterator(db->resources);
    LocalResource *local;
    UCX_MAP_FOREACH(key, local, i) {
        if (res_matches_filter(dir, local->path)) {
            continue;
        }
        // sync_remove_resource does all necessary tests
        sync_remove_local_resource(dir, local);
    }
    ucx_map_free(db->resources);
    db->resources = svrres;
    
    // TODO: cleanup - BUT DONT CLEANUP SYNC CONFIG (do this in main!)
    
    // store db
    if(store_db(db, dir->database)) {
        fprintf(stderr, "Cannot store sync db\n");
        return -1;
    }
    
    return 0;
}

int sync_get_resource(CmdArgs *a, SyncDirectory *dir, DavResource *res, SyncDatabase *db) {
    int cdt = cmd_getoption(a, "conflict") ? 0 : 1; // conflict detection
    
    LocalResource *local = ucx_map_cstr_get(db->resources, res->path);
    char *local_path = util_concat_path(dir->path, res->path);
    
    char *etag = dav_get_property(res, "D:getetag");
    struct stat s;
    if(local) {
        int exists = 1;
        if(stat(local_path, &s)) {
            // Ignore the fact, that the file is locally removed. If the
            // server has an updated version, we readd the file or the
            // next push will delete it on the server.
            if(errno != ENOENT) {
                fprintf(stderr, "Cannot stat file: %s\n", local_path);
                free(local_path);
                return -1;
            } else {
                exists = 0;
            }
        }
              
        if(local->etag) {
            sstr_t e = sstr(etag);
            if(sstrprefix(e, S("W/"))) {
                e = sstrsubs(e, 2);
            }
            if(!strcmp(e.ptr, local->etag)) {
                // resource is already up-to-date on the client
                return 0;
            }
        }
        
        if(cdt && exists && s.st_mtime != local->last_modified) {
            // file modified on the server and on the client
            rename_local_file(dir, db, local->path);
        }
    } else {
        if(stat(local_path, &s)) {
            if(errno != ENOENT) {
                fprintf(stderr, "Cannot stat file: %s\n", local_path);
            }
        } else if(S_ISDIR(s.st_mode)) {
            //fprintf(stderr, "Error: file %s is a directory\n", local_path);
        } else if(cdt) {
            // rename file on conflict
            rename_local_file(dir, db, res->path);
        }
    }
      
    int ret = 0;
    if(res->iscollection) {
        mode_t mode = S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH;
        if(util_mkdir(local_path, mode) && errno != EEXIST) {
            ret = -1;
        }
    } else {
        FILE *out = fopen(local_path, "wb");
        if(!out) {
            fprintf(stderr, "Cannot open output file: %s\n", local_path);
            free(local_path);
            return -1;
        }
        printf("get: %s\n", res->path);
        if(dav_get_content(res, out, (dav_write_func)fwrite)) {
            ret = -1;
        }
        fclose(out);
        
        if(stat(local_path, &s)) {
            fprintf(stderr, "Cannot stat file: %s\n", local_path);
        }
        
        if(ret == 0) {
            if(!local) {
                // new local resource
                local = calloc(1, sizeof(LocalResource));
                local->path = strdup(res->path);
                ucx_map_cstr_put(db->resources, local->path, local);
            }
            
            if(local->etag) {
                free(local->etag);
            }
            // set metadata from stat
            local->etag = etag;
            local->last_modified = s.st_mtime;
            local->size = s.st_size;
        }
    }
    
    free(local_path);
    return ret;
}

void sync_remove_local_resource(SyncDirectory *dir, LocalResource *res) {
    char *local_path = util_concat_path(dir->path, res->path);
    struct stat s;
    if(stat(local_path, &s)) {
        free(local_path);
        return;
    }
    
    if(s.st_mtime != res->last_modified) {
        free(local_path);
        return;
    }
    
    printf("delete: %s\n", res->path);
    
    if(dir->trash) {
        move_to_trash(dir, local_path);
    } else if(unlink(local_path)) {
        fprintf(stderr, "Cannot remove file %s\n", local_path);
    }
    free(local_path);
}

void rename_local_file(SyncDirectory *dir, SyncDatabase *db, char *path) {
    char *local_path = util_concat_path(dir->path, path);
    char *parent = util_parent_path(local_path);
    
    int rev = 0;
    struct stat s;
    int loop = 1;
    do {
        sstr_t new_path = ucx_asprintf(
        ucx_default_allocator(),
            "%sorig.%d.%s",
            parent,
            rev,
            util_resource_name(path));
        
        
        if(stat(new_path.ptr, &s)) {
            if(errno == ENOENT) {
                loop = 0;
                printf("conflict: %s\n", local_path);
                if(rename(local_path, new_path.ptr)) {
                    //printf("errno: %d\n", errno);
                    fprintf(
                            stderr,
                            "Cannot rename file %s to %s\n",
                            local_path,
                            new_path.ptr);
                }
            }
        }
        rev++;
        free(new_path.ptr);
    } while(loop);
    free(parent);
}

void move_to_trash(SyncDirectory *dir, char *path) {
    char *new_path = NULL;
    for (int i=0;;i++) {
        sstr_t np = ucx_asprintf(
        ucx_default_allocator(),
            "%s%d-%s",
            dir->trash,
            i,
            util_resource_name(path));
        
        struct stat s;
        if(stat(np.ptr, &s)) {
            if(errno == ENOENT) {
                new_path = np.ptr;
            }
            break;
        }
        free(np.ptr);
    };
    
    if(!new_path) {
        fprintf(stderr, "Cannot move file %s to trash.\n", path);
        return;
    }
    
    if(rename(path, new_path)) {
        //printf("errno: %d\n", errno);
        fprintf(
                stderr,
                "Cannot rename file %s to %s\n",
                path,
                new_path);
    }
    
    free(new_path);
}

int cmd_push(CmdArgs *a) {
    if(a->argc != 1) {
        fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many");
        return -1;
    }
    
    SyncDirectory *dir = scfg_get_dir(a->argv[0]);
    if(!dir) {
        fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]);
        return -1;
    }
    
    Repository *repo = get_repository(sstr(dir->repository));
    if(!repo) {
        fprintf(stderr, "Unkown repository %s\n", dir->name);
        return -1;
    }
    
    SyncDatabase *db = load_db(dir->database);
    if(!db) {
        fprintf(stderr, "Cannot load database file: %s\n", dir->database);
        return -1;
    }
    
    char *new_url = NULL;
    if(dir->collection) {
        new_url = util_concat_path(repo->url, dir->collection);
    }
    DavSession *sn = create_session(ctx, repo, new_url ? new_url : repo->url);
    if(new_url) {
        free(new_url);
    }
    if (cmd_getoption(a, "verbose")) {
        curl_easy_setopt(sn->handle, CURLOPT_VERBOSE, 1L);
        curl_easy_setopt(sn->handle, CURLOPT_STDERR, stderr);
    }
    
    // upload all changed files
    UcxList *resources = cmd_getoption(a, "read") ?
            read_changes(dir, db) : local_scan(dir, db);
    
    UcxMap *lclres = ucx_map_new(db->resources->count);
    UCX_FOREACH(elm, resources) {
        LocalResource *local_res = elm->data;
        if (!res_matches_filter(dir, local_res->path+1)) {
            // upload every changed file
            if (local_resource_is_changed(dir, db, local_res)) {
                printf("put: %s\n", local_res->path);
                DavResource *res = dav_resource_new(sn, local_res->path);
                if(sync_put_resource(dir, res, local_res)) {
                    // TODO: I don't know what to do now
                }
                dav_resource_free(res);
            }
            
            // remove every locally available resource from db->resource
            // the remaining elements are all deleted files
            ucx_map_cstr_put(lclres, local_res->path, local_res);
            ucx_map_cstr_remove(db->resources, local_res->path); // TODO: element leaked
        }
    }
    ucx_list_free(resources);
    
    // delete all removed files
    UcxMapIterator i = ucx_map_iterator(db->resources);
    LocalResource *local;
    UCX_MAP_FOREACH(key, local, i) {
        if (!res_matches_filter(dir, local->path+1)) {
            if(sync_delete_remote_resource(sn, local)) {
                ucx_map_cstr_put(lclres, local->path, local);
            }
        }
    }
    ucx_map_free(db->resources);
    db->resources = lclres;
    
    // TODO: free res
    
    // store db
    if(store_db(db, dir->database)) {
        fprintf(stderr, "Cannot store sync db\n");
        return -1;
    }
    
    return 0;
}

UcxList* local_scan(SyncDirectory *dir, SyncDatabase *db) {
    UcxList *resources = NULL;
    
    char *path = strdup("/");
    UcxList *stack = ucx_list_prepend(NULL, path);
    while(stack) {
        // get a directory path from the stack and read all entries
        // if an entry is a directory, put it on the stack
        
        char *p = stack->data;
        stack = ucx_list_remove(stack, stack);
        char *local_path = util_concat_path(dir->path, p);
        DIR *local_dir = opendir(local_path);
        
        if(!local_dir) {
            fprintf(stderr, "Cannot open directory %s\n", local_path);
        } else {
            struct dirent *ent;
            while((ent = readdir(local_dir)) != NULL) {
                if(!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) {
                    continue;
                }
                
                char *new_path = util_concat_path(p, ent->d_name);
                int isdir;
                LocalResource *res = local_resource_new(dir, db, new_path, &isdir);
                if(isdir) {
                    stack = ucx_list_prepend(stack, new_path);
                } else if(res) {
                    resources = ucx_list_append(resources, res);
                    free(new_path);
                } else {
                    free(new_path);
                }
            }
            closedir(local_dir);
 
        }
        free(local_path);
        free(p);
    }
    
    return resources;
}

UcxList* read_changes(SyncDirectory *dir, SyncDatabase *db) {
    UcxProperties *parser = ucx_properties_new();
    parser->delimiter = ':';
    
    UcxList *resources = NULL;
    sstr_t name;
    sstr_t value;
    
    char buf[STDIN_BUF_SIZE];
    size_t r;
    while(!feof(stdin)) {
        r = fread(buf, 1, STDIN_BUF_SIZE, stdin);
        ucx_properties_fill(parser, buf, r);
        while(ucx_properties_next(parser, &name, &value)) {
            if(value.length == 0) {
                fprintf(stderr, "Wrong input\n");
                continue;
            }
            if(value.ptr[0] == '"'
                    && value.length > 2
                    && value.ptr[value.length - 1] == '"')
            {
                value.ptr[value.length - 1] = '\0';
                value.ptr++;
                value.length -= 2;
            }
            value = sstrdup(value);
            
            if(!sstrcmp(name, S("put"))) {
                int isdir;
                LocalResource *res = local_resource_new(dir, db, value.ptr, &isdir);
                if(res) {
                    resources = ucx_list_append(resources, res);
                }
            } else if(!sstrcmp(name, S("remove"))) {
                LocalResource *res = calloc(1, sizeof(LocalResource));
                res->path = sstrdup(value).ptr;
                if(res) {
                    ucx_map_sstr_put(db->remove, value, res);
                    ucx_map_sstr_remove(db->resources, value);
                }
                
            }
            
            free(value.ptr);
        }
    }
    ucx_properties_free(parser);    
    
    return resources;
}

LocalResource* local_resource_new(SyncDirectory *dir, SyncDatabase *db, char *path, int *isdir) {
    char *file_path = util_concat_path(dir->path, path);
    struct stat s;
    if(stat(file_path, &s)) {
        fprintf(stderr, "Cannot stat file %s\n", file_path);
        free(file_path);
        return NULL;
    }
    free(file_path);

    if(!S_ISDIR(s.st_mode)) {
        *isdir = 0;
        LocalResource *res = calloc(1, sizeof(LocalResource));
        res->path = strdup(path);
        res->etag = NULL;
        res->last_modified = s.st_mtime;
        res->size = s.st_size;
        return res;
    } else {
        *isdir = 1;
    }
    return NULL;
}

int local_resource_is_changed(SyncDirectory *dir, SyncDatabase *db, LocalResource *res) {
    LocalResource *db_res = ucx_map_cstr_get(db->resources, res->path);
    if(db_res) {
        if(db_res->etag) {
            res->etag = strdup(db_res->etag);
        }
        
        if(db_res->last_modified == res->last_modified && db_res->size == res->size) {
            return 0;
        }
    }
    return 1;
}


int sync_put_resource(SyncDirectory *dir, DavResource *res, LocalResource *local) {
    char *local_path = util_concat_path(dir->path, res->path);
    FILE *in = fopen(local_path, "rb");
    if(!in) {
        fprintf(stderr, "Cannot open file %s\n", local_path);
        free(local_path);
        return -1;
    }
    free(local_path);
    
    dav_set_content(res, in, (dav_read_func)fread);
    
    int ret = -1;
    for(;;) {
        if(dav_create(res)) {
            break;
        }
        if(dav_store(res)) {
            break;
        }
        ret = 0;
        break;
    }
    
    if(ret == 0) {      
        // get new etag
        DavResource *up_res = dav_get(res->session, res->path, "D:getetag");
        char *etag = dav_get_property(up_res, "D:getetag");
        if(etag) {
            if(strlen(etag) > 2 && etag[0] == 'W' && etag[1] == '/') {
                etag = etag + 2;
            } 
        }
        
        if(local->etag) {
            free(local->etag);
        }
        
        if(etag) {
            local->etag = strdup(etag);
        } else {
            local->etag = NULL;
        }
        dav_resource_free(up_res);
    }
    
    fclose(in);
    
    return ret;
}

int sync_delete_remote_resource(DavSession *sn, LocalResource *local_res) {
    DavResource *res = dav_get(sn, local_res->path, "D:getetag");
    if(!res) {
        return sn->error == DAV_NOT_FOUND ? 0 : 1;
    }
    
    char *etag = dav_get_property(res, "D:getetag");
    if(etag) {
        if(strlen(etag) > 2 && etag[0] == 'W' && etag[1] == '/') {
            etag = etag + 2;
        } 
    }
    
    int ret = 0;
    if(etag && !strcmp(etag, local_res->etag)) {
        // local resource metadata == remote resource metadata
        // resource can be deleted
        printf("delete: %s\n", res->path);
        if(dav_delete(res)) {
            if(sn->error != DAV_NOT_FOUND) {
                fprintf(stderr, "Cannot delete resource %s\n", res->path);
            }
        }
    } else {
        ret = 1;
    }
    
    // cleanup
    dav_resource_free(res);
    
    return ret;
}