Fri, 04 Jul 2014 12:09:48 +0200
dav-sync deletes removed files
/* * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. * * Copyright 2013 Olaf Wintermann. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include <stdio.h> #include <stdlib.h> #include <string.h> #include <errno.h> #include <unistd.h> #include <time.h> #include <libxml/xmlerror.h> #include <sys/types.h> #include <ucx/string.h> #include <dirent.h> #include <libidav/webdav.h> #include <libidav/utils.h> #include "config.h" #include "scfg.h" #include "sopt.h" #include "db.h" #include "sync.h" static DavContext *ctx; static void xmlerrorfnc(void * c, const char * msg, ... ) { // nothing } int main(int argc, char **argv) { if(argc < 2) { fprintf(stderr, "Missing command\n"); print_usage(argv[0]); return -1; } char *cmd = argv[1]; CmdArgs *args = cmd_parse_args(argc - 2, argv + 2); if(!args) { print_usage(argv[0]); return -1; } xmlGenericErrorFunc fnc = xmlerrorfnc; initGenericErrorDefaultFunc(&fnc); ctx = dav_context_new(); load_config(ctx); if(load_sync_config()) { return EXIT_FAILURE; } int ret = EXIT_FAILURE; if(!strcmp(cmd, "pull")) { ret = cmd_pull(args); } else if(!strcmp(cmd, "push")) { ret = cmd_push(args); } else if(!strcmp(cmd, "sync")) { ret = cmd_sync(args); } return ret; } void print_usage(char *cmd) { } int cmd_pull(CmdArgs *a) { if(a->argc != 1) { fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many"); return -1; } SyncDirectory *dir = scfg_get_dir(a->argv[0]); if(!dir) { fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]); return -1; } Repository *repo = get_repository(sstr(dir->repository)); if(!repo) { fprintf(stderr, "Unkown repository %s\n", dir->name); return -1; } SyncDatabase *db = load_db(dir->database); if(!db) { fprintf(stderr, "Cannot load database file: %s\n", dir->database); return -1; } DavSession *sn = dav_session_new_auth( ctx, repo->url, repo->user, repo->password); dav_session_set_flags(sn, get_repository_flags(repo)); sn->key = dav_context_get_key(ctx, repo->default_key); DavResource *ls = dav_query(sn, "get D:getetag from / where lastmodified > 0 with depth -1"); if(!ls) { fprintf(stderr, "Error\n"); // TODO: free return -1; } if(!ls->children) { // TODO: free return 0; // empty repository } UcxMap *svrres = ucx_map_new(db->resources->count); UcxList *stack = ucx_list_prepend(NULL, ls->children); while(stack) { DavResource *res = stack->data; stack = ucx_list_remove(stack, stack); while(res) { // download the resource if(sync_get_resource(dir, res, db)) { fprintf(stderr, "sync_get_resource failed for resource: %s\n", res->path); } LocalResource *local = ucx_map_cstr_get(db->resources, res->path); ucx_map_cstr_put(svrres, res->path, local); ucx_map_cstr_remove(db->resources, res->path); if(res->children) { stack = ucx_list_prepend(stack, res->children); } res = res->next; } } UcxMapIterator i = ucx_map_iterator(db->resources); LocalResource *local; UCX_MAP_FOREACH(key, local, i) { sync_remove_resource(dir, local); } ucx_map_free(db->resources); db->resources = svrres; // TODO: cleanup // store db if(store_db(db, dir->database)) { fprintf(stderr, "Cannot store sync db\n"); return -1; } return 0; } int sync_get_resource(SyncDirectory *dir, DavResource *res, SyncDatabase *db) { LocalResource *removed = ucx_map_cstr_get(db->remove, res->path); if(removed) { return 0; } LocalResource *local = ucx_map_cstr_get(db->resources, res->path); char *local_path = util_concat_path(dir->path, res->path); char *etag = dav_get_property(res, "D:getetag"); struct stat s; if(local) { if(stat(local_path, &s)) { if(errno == ENOENT) { printf("removed %s\n", res->path); // the file is in the database, but doesn't exists // mark the file as removed to delete it on next push ucx_map_cstr_remove(db->resources, local->path); ucx_map_cstr_put(db->remove, local->path, local); return 0; } else { fprintf(stderr, "stat failed: %s\n", local_path); free(local_path); return -1; } } if(local->etag) { sstr_t e = sstr(etag); if(sstrprefix(e, S("W/"))) { e = sstrsubs(e, 2); } if(!strcmp(e.ptr, local->etag)) { // resource is already up-to-date on the client return 0; } } } int ret = 0; if(res->iscollection) { mode_t mode = S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH; //printf("mkdir %s\n", local_path); if(util_mkdir(local_path, mode) && errno != EEXIST) { ret = -1; } } else { FILE *out = fopen(local_path, "w"); if(!out) { fprintf(stderr, "cannot open output file: %s\n", local_path); free(local_path); return -1; } printf("get %s\n", res->path); if(dav_get_content(res, out, (dav_write_func)fwrite)) { ret = -1; } fclose(out); if(ret == 0) { if(!local) { // new local resource local = calloc(1, sizeof(LocalResource)); local->path = strdup(res->path); ucx_map_cstr_put(db->resources, local->path, local); } if(local->etag) { free(local->etag); } // set metadata from stat local->etag = etag; local->last_modified = s.st_mtim.tv_sec; local->size = s.st_size; } } free(local_path); return ret; } void sync_remove_resource(SyncDirectory *dir, LocalResource *res) { char *local_path = util_concat_path(dir->path, res->path); struct stat s; if(stat(local_path, &s)) { free(local_path); return; } if(s.st_mtim.tv_sec != res->last_modified) { free(local_path); return; } printf("delete: %s\n", res->path); if(unlink(local_path)) { fprintf(stderr, "Cannot remove file %s\n", local_path); } free(local_path); } int cmd_push(CmdArgs *a) { if(a->argc != 1) { fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many"); return -1; } SyncDirectory *dir = scfg_get_dir(a->argv[0]); if(!dir) { fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]); return -1; } Repository *repo = get_repository(sstr(dir->repository)); if(!repo) { fprintf(stderr, "Unkown repository %s\n", dir->name); return -1; } SyncDatabase *db = load_db(dir->database); if(!db) { fprintf(stderr, "Cannot load database file: %s\n", dir->database); return -1; } DavSession *sn = dav_session_new_auth( ctx, repo->url, repo->user, repo->password); dav_session_set_flags(sn, get_repository_flags(repo)); sn->key = dav_context_get_key(ctx, repo->default_key); // upload all changed files UcxList *resources = local_scan(dir, db); UCX_FOREACH(elm, resources) { char *path = elm->data; printf("put: %s\n", path); DavResource *res = dav_resource_new(sn, path); sync_put_resource(dir, res, db); dav_resource_free(res); free(path); } ucx_list_free(resources); // delete all removed files UcxMapIterator i = ucx_map_iterator(db->remove); LocalResource *local; UCX_MAP_FOREACH(key, local, i) { DavResource *res = dav_resource_new(sn, local->path); printf("delete: %s\n", res->path); if(dav_delete(res)) { if(sn->error != DAV_NOT_FOUND) { fprintf(stderr, "Cannot delete resource %s\n", res->path); } } dav_resource_free(res); // TODO: free local resource ucx_map_remove(db->remove, key); } // TODO: free res // store db if(store_db(db, dir->database)) { fprintf(stderr, "Cannot store sync db\n"); return -1; } return 0; } UcxList* local_scan(SyncDirectory *dir, SyncDatabase *db) { UcxList *resources = NULL; char *path = strdup("/"); UcxList *stack = ucx_list_prepend(NULL, path); while(stack) { // get a directory path from the stack and read all entries // if an entry is a directory, put it on the stack // otherwise compare the metadata with the db content char *p = stack->data; stack = ucx_list_remove(stack, stack); char *local_path = util_concat_path(dir->path, p); DIR *local_dir = opendir(local_path); if(!local_dir) { fprintf(stderr, "Cannot open directory %s\n", local_path); } else { long namemax = namemax = pathconf(path, _PC_NAME_MAX); if(namemax == 0) { namemax = 255; } struct dirent *ent = malloc(sizeof(struct dirent) + namemax + 1); struct dirent *res = NULL; while(!readdir_r(local_dir, ent, &res) && res) { if(!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) { continue; } char *new_path = util_concat_path(p, ent->d_name); char *file_path = util_concat_path(dir->path, new_path); struct stat s; if(stat(file_path, &s)) { fprintf(stderr, "Cannot stat file %s\n", file_path); free(new_path); free(file_path); continue; } free(file_path); if(S_ISDIR(s.st_mode)) { stack = ucx_list_prepend(stack, new_path); } else { LocalResource *res = ucx_map_cstr_get( db->resources, new_path); if(res) { // the file is already in the database // compare length and lastmodified date if(res->last_modified == s.st_mtim.tv_sec && res->size == s.st_size) { // skip this file free(new_path); } else { // add file to list resources = ucx_list_append( resources, new_path); // update db entries res->size = s.st_size; res->last_modified = s.st_mtim.tv_sec; } } else { // add file to list LocalResource *res = calloc(1, sizeof(LocalResource)); res->path = strdup(new_path); res->etag = NULL; res->last_modified = s.st_mtim.tv_sec; res->size = s.st_size; ucx_map_cstr_put(db->resources, res->path, res); resources = ucx_list_append(resources, new_path); } } } closedir(local_dir); free(ent); } free(local_path); free(p); } return resources; } int sync_put_resource(SyncDirectory *dir, DavResource *res, SyncDatabase *db) { char *local_path = util_concat_path(dir->path, res->path); FILE *in = fopen(local_path, "r"); if(!in) { fprintf(stderr, "Cannot open file %s\n", local_path); free(local_path); return -1; } free(local_path); dav_set_content(res, in, (dav_read_func)fread); int ret = -1; for(;;) { if(dav_create(res)) { break; } if(dav_store(res)) { break; } ret = 0; break; } if(ret == 0) { LocalResource *local_res = ucx_map_cstr_get(db->resources, res->path); if(local_res->etag) { free(local_res->etag); } DavResource *up_res = dav_get(res->session, res->path, "D:getetag"); char *etag = dav_get_property(up_res, "D:getetag"); if(etag) { if(strlen(etag) > 2 && etag[0] == 'W' && etag[1] == '/') { etag = etag + 2; } } if(etag) { local_res->etag = strdup(etag); } else { local_res->etag = NULL; } } fclose(in); return 0; } int cmd_sync(CmdArgs *a) { return 0; }