dav/sync.c

/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 2013 Olaf Wintermann. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *   1. Redistributions of source code must retain the above copyright
 *      notice, this list of conditions and the following disclaimer.
 *
 *   2. Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in the
 *      documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <unistd.h>
#include <time.h>
#include <libxml/xmlerror.h>
#include <sys/types.h>
#include <ucx/string.h>
#include <ucx/utils.h>
#include <dirent.h>

#include <libidav/webdav.h>
#include <libidav/utils.h>

#include "config.h"
#include "scfg.h"
#include "sopt.h"
#include "db.h"

#include "sync.h"

static DavContext *ctx;

static void xmlerrorfnc(void * c, const char * msg, ... ) {
    // nothing
}

int main(int argc, char **argv) {
    if(argc < 2) {
        fprintf(stderr, "Missing command\n");
        print_usage(argv[0]);
        return -1;
    }
    
    char *cmd = argv[1];
    CmdArgs *args = cmd_parse_args(argc - 2, argv + 2);
    if(!args) {
        print_usage(argv[0]);
        return -1;
    }
    
    xmlGenericErrorFunc fnc = xmlerrorfnc;
    initGenericErrorDefaultFunc(&fnc);
    ctx = dav_context_new();
    load_config(ctx);
    if(load_sync_config()) {
        return EXIT_FAILURE;
    }
    
    int ret = EXIT_FAILURE;
    if(!strcmp(cmd, "pull")) {
        ret = cmd_pull(args);
    } else if(!strcmp(cmd, "push")) {
        ret = cmd_push(args);
    } else if(!strcmp(cmd, "sync")) {
        ret = cmd_sync(args);
    }
    
    return ret;
}

void print_usage(char *cmd) {
    
}

int cmd_pull(CmdArgs *a) {
    if(a->argc != 1) {
        fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many");
        return -1;
    }
    
    SyncDirectory *dir = scfg_get_dir(a->argv[0]);
    if(!dir) {
        fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]);
        return -1;
    }
    
    Repository *repo = get_repository(sstr(dir->repository));
    if(!repo) {
        fprintf(stderr, "Unkown repository %s\n", dir->name);
        return -1;
    }
    
    SyncDatabase *db = load_db(dir->database);
    if(!db) {
        fprintf(stderr, "Cannot load database file: %s\n", dir->database);
        return -1;
    }
    
    DavSession *sn = dav_session_new_auth(
            ctx,
            repo->url,
            repo->user,
            repo->password);
    dav_session_set_flags(sn, get_repository_flags(repo));
    sn->key = dav_context_get_key(ctx, repo->default_key);
    
    DavResource *ls = dav_query(sn, "get D:getetag from / where lastmodified > 0 with depth -1");
    if(!ls) {
        fprintf(stderr, "Error\n");
        // TODO: free
        return -1;
    }
    
    if(!ls->children) {
        // TODO: free
        return 0; // empty repository
    }
    
    UcxMap *svrres = ucx_map_new(db->resources->count);
    
    UcxList *stack = ucx_list_prepend(NULL, ls->children);
    while(stack) {
        DavResource *res = stack->data;
        stack = ucx_list_remove(stack, stack);
         
        while(res) { 
            // download the resource
            if(sync_get_resource(dir, res, db)) {
                fprintf(stderr, "sync_get_resource failed for resource: %s\n", res->path);
            }
            
            LocalResource *local = ucx_map_cstr_get(db->resources, res->path);
            ucx_map_cstr_put(svrres, res->path, local);
            ucx_map_cstr_remove(db->resources, res->path);
            
            if(res->children) {
                stack = ucx_list_prepend(stack, res->children);
            }
            res = res->next;
        }
    }
    
    UcxMapIterator i = ucx_map_iterator(db->resources);
    LocalResource *local;
    UCX_MAP_FOREACH(key, local, i) {
        sync_remove_resource(dir, local);
    }
    ucx_map_free(db->resources);
    db->resources = svrres;
    
    // TODO: cleanup
    
    // store db
    if(store_db(db, dir->database)) {
        fprintf(stderr, "Cannot store sync db\n");
        return -1;
    }
    
    return 0;
}

int sync_get_resource(SyncDirectory *dir, DavResource *res, SyncDatabase *db) {
    LocalResource *removed = ucx_map_cstr_get(db->remove, res->path);
    if(removed) {
        return 0;
    }
    
    LocalResource *local = ucx_map_cstr_get(db->resources, res->path);
    char *local_path = util_concat_path(dir->path, res->path);
    
    char *etag = dav_get_property(res, "D:getetag");
    struct stat s;
    if(local) {
        if(stat(local_path, &s)) {
            if(errno == ENOENT) {
                printf("removed: %s\n", res->path);
                // the file is in the database, but doesn't exists
                // mark the file as removed to delete it on next push
                ucx_map_cstr_remove(db->resources, local->path);
                ucx_map_cstr_put(db->remove, local->path, local);
                return 0;
            } else {
                fprintf(stderr, "Cannot stat file: %s\n", local_path);
                free(local_path);
                return -1;
            }
        }
              
        if(local->etag) {
            sstr_t e = sstr(etag);
            if(sstrprefix(e, S("W/"))) {
                e = sstrsubs(e, 2);
            }
            if(!strcmp(e.ptr, local->etag)) {
                // resource is already up-to-date on the client
                return 0;
            }
        }
        
        if(s.st_mtim.tv_sec != local->last_modified) {
            // file modified on the server and on the client
            rename_local_file(dir, db, local->path);
        }
    } else {
        if(stat(local_path, &s)) {
            if(errno != ENOENT) {
                fprintf(stderr, "Cannot stat file: %s\n", local_path);
            }
        } else if(!S_ISDIR(s.st_mode)) {
            rename_local_file(dir, db, res->path);
        }
    }
      
    int ret = 0;
    if(res->iscollection) {
        mode_t mode = S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH;
        //printf("mkdir %s\n", local_path);
        if(util_mkdir(local_path, mode) && errno != EEXIST) {
            ret = -1;
        }
    } else {
        FILE *out = fopen(local_path, "w");
        if(!out) {
            fprintf(stderr, "Cannot open output file: %s\n", local_path);
            free(local_path);
            return -1;
        }
        printf("get: %s\n", res->path);
        if(dav_get_content(res, out, (dav_write_func)fwrite)) {
            ret = -1;
        }
        fclose(out);
        
        if(stat(local_path, &s)) {
            fprintf(stderr, "Cannot stat file: %s\n", local_path);
        }
        
        if(ret == 0) {
            if(!local) {
                // new local resource
                local = calloc(1, sizeof(LocalResource));
                local->path = strdup(res->path);
                ucx_map_cstr_put(db->resources, local->path, local);
            }
            
            if(local->etag) {
                free(local->etag);
            }
            // set metadata from stat
            local->etag = etag;
            local->last_modified = s.st_mtim.tv_sec;
            local->size = s.st_size;
        }
    }
    
    free(local_path);
    return ret;
}

void sync_remove_resource(SyncDirectory *dir, LocalResource *res) {
    char *local_path = util_concat_path(dir->path, res->path);
    struct stat s;
    if(stat(local_path, &s)) {
        free(local_path);
        return;
    }
    
    if(s.st_mtim.tv_sec != res->last_modified) {
        free(local_path);
        return;
    }
    
    printf("delete: %s\n", res->path);
    if(unlink(local_path)) {
        fprintf(stderr, "Cannot remove file %s\n", local_path);
    }
    free(local_path);
}

void rename_local_file(SyncDirectory *dir, SyncDatabase *db, char *path) {
    char *local_path = util_concat_path(dir->path, path);
    char *parent = util_parent_path(local_path);
    
    int rev = 0;
    struct stat s;
    int loop = 1;
    do {
        sstr_t new_path = ucx_asprintf(
        ucx_default_allocator(),
            "%sorig.%d.%s",
            parent,
            rev,
            util_resource_name(path));
        
        
        if(stat(new_path.ptr, &s)) {
            if(errno == ENOENT) {
                loop = 0;
                printf("conflict: %s\n", local_path);
                if(rename(local_path, new_path.ptr)) {
                    printf("errno: %d\n", errno);
                    fprintf(
                            stderr,
                            "Cannot rename file %s to %s\n",
                            local_path,
                            new_path.ptr);
                }
            }
        }
        rev++;
        free(new_path.ptr);
    } while(loop);
    free(parent);
}

int cmd_push(CmdArgs *a) {
    if(a->argc != 1) {
        fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many");
        return -1;
    }
    
    SyncDirectory *dir = scfg_get_dir(a->argv[0]);
    if(!dir) {
        fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]);
        return -1;
    }
    
    Repository *repo = get_repository(sstr(dir->repository));
    if(!repo) {
        fprintf(stderr, "Unkown repository %s\n", dir->name);
        return -1;
    }
    
    SyncDatabase *db = load_db(dir->database);
    if(!db) {
        fprintf(stderr, "Cannot load database file: %s\n", dir->database);
        return -1;
    }
    
    DavSession *sn = dav_session_new_auth(
            ctx,
            repo->url,
            repo->user,
            repo->password);
    dav_session_set_flags(sn, get_repository_flags(repo));
    sn->key = dav_context_get_key(ctx, repo->default_key);
    
    // upload all changed files
    UcxList *resources = local_scan(dir, db);
    UCX_FOREACH(elm, resources) {
        char *path = elm->data;
        printf("put: %s\n", path);
        DavResource *res = dav_resource_new(sn, path);
        sync_put_resource(dir, res, db);
        dav_resource_free(res);
        free(path);
    }
    ucx_list_free(resources);
    
    // delete all removed files
    UcxMapIterator i = ucx_map_iterator(db->remove);
    LocalResource *local;
    UCX_MAP_FOREACH(key, local, i) {
        DavResource *res = dav_resource_new(sn, local->path);
        printf("delete: %s\n", res->path);
        if(dav_delete(res)) {
            if(sn->error != DAV_NOT_FOUND) {
                fprintf(stderr, "Cannot delete resource %s\n", res->path);
            }
        }
        dav_resource_free(res);
        // TODO: free local resource
        ucx_map_remove(db->remove, key);
    }
    
    // TODO: free res
    
    // store db
    if(store_db(db, dir->database)) {
        fprintf(stderr, "Cannot store sync db\n");
        return -1;
    }
    
    return 0;
}

UcxList* local_scan(SyncDirectory *dir, SyncDatabase *db) {
    UcxList *resources = NULL;
    
    char *path = strdup("/");
    UcxList *stack = ucx_list_prepend(NULL, path);
    while(stack) {
        // get a directory path from the stack and read all entries
        // if an entry is a directory, put it on the stack
        // otherwise compare the metadata with the db content
        
        char *p = stack->data;
        stack = ucx_list_remove(stack, stack);
        char *local_path = util_concat_path(dir->path, p);
        DIR *local_dir = opendir(local_path);
        
        if(!local_dir) {
            fprintf(stderr, "Cannot open directory %s\n", local_path);
        } else {
            long namemax = namemax = pathconf(path, _PC_NAME_MAX);
            if(namemax == 0) {
                namemax = 255;
            }
            struct dirent *ent = malloc(sizeof(struct dirent) + namemax + 1);
            struct dirent *res = NULL;
            while(!readdir_r(local_dir, ent, &res) && res) {
                if(!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) {
                    continue;
                }
                
                char *new_path = util_concat_path(p, ent->d_name);
                char *file_path = util_concat_path(dir->path, new_path);
                struct stat s;
                if(stat(file_path, &s)) {
                    fprintf(stderr, "Cannot stat file %s\n", file_path);
                    free(new_path);
                    free(file_path);
                    continue;
                }
                free(file_path);
                
                if(S_ISDIR(s.st_mode)) {
                    stack = ucx_list_prepend(stack, new_path);
                } else {
                    LocalResource *res = ucx_map_cstr_get(
                            db->resources,
                            new_path);
                    if(res) {
                        // the file is already in the database
                        // compare length and lastmodified date
                        
                        if(res->last_modified == s.st_mtim.tv_sec
                                && res->size == s.st_size)
                        {
                            // skip this file
                            free(new_path);
                        } else {
                            // add file to list
                            resources = ucx_list_append(
                                    resources,
                                    new_path);
                            
                            // update db entries
                            res->size = s.st_size;
                            res->last_modified = s.st_mtim.tv_sec;
                        }                       
                    } else {
                        // add file to list
                        LocalResource *res = calloc(1, sizeof(LocalResource));
                        res->path = strdup(new_path);
                        res->etag = NULL;
                        res->last_modified = s.st_mtim.tv_sec;
                        res->size = s.st_size;
                        ucx_map_cstr_put(db->resources, res->path, res);
                        resources = ucx_list_append(resources, new_path);
                    }
                }
            }
            closedir(local_dir);
            free(ent);
        }
        free(local_path);
        free(p);
    }
    
    return resources;
}

int sync_put_resource(SyncDirectory *dir, DavResource *res, SyncDatabase *db) {
    char *local_path = util_concat_path(dir->path, res->path);
    FILE *in = fopen(local_path, "r");
    if(!in) {
        fprintf(stderr, "Cannot open file %s\n", local_path);
        free(local_path);
        return -1;
    }
    free(local_path);
    
    dav_set_content(res, in, (dav_read_func)fread);
    
    int ret = -1;
    for(;;) {
        if(dav_create(res)) {
            break;
        }
        if(dav_store(res)) {
            break;
        }
        ret = 0;
        break;
    }
    
    if(ret == 0) {
        LocalResource *local_res = ucx_map_cstr_get(db->resources, res->path);
        if(local_res->etag) {
            free(local_res->etag);
        }
        
        DavResource *up_res = dav_get(res->session, res->path, "D:getetag");
        char *etag = dav_get_property(up_res, "D:getetag");
        if(etag) {
            if(strlen(etag) > 2 && etag[0] == 'W' && etag[1] == '/') {
                etag = etag + 2;
            } 
        }
        
        
        if(etag) {
            local_res->etag = strdup(etag);
        } else {
            local_res->etag = NULL;
        }
    }
    
    fclose(in);
    
    return 0;
}


int cmd_sync(CmdArgs *a) {
    return 0;
}