Wed, 20 Mar 2019 09:27:29 +0100
changes list-versions output
/* * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. * * Copyright 2018 Olaf Wintermann. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include <stdio.h> #include <stdlib.h> #include <string.h> #include <errno.h> #include <unistd.h> #include <signal.h> #include <time.h> #include <utime.h> #include <libxml/xmlerror.h> #include <sys/types.h> #include <ucx/string.h> #include <ucx/utils.h> #include <ucx/properties.h> #include <dirent.h> #include <libidav/webdav.h> #include <libidav/utils.h> #include <libidav/crypto.h> #include <libidav/session.h> #include "sync.h" #include "config.h" #include "sopt.h" #include "error.h" #include "assistant.h" #include "libxattr.h" #include "tags.h" #include "system.h" #include <pthread.h> #include <ctype.h> static DavContext *ctx; static int sync_shutdown = 0; static void xmlerrorfnc(void * c, const char * msg, ... ) { va_list ap; va_start(ap, msg); vfprintf(stderr, msg, ap); va_end(ap); } /* * strcmp version that works with NULL pointers */ static int nullstrcmp(const char *s1, const char *s2) { if(!s1 && s2) { return -1; } if(s1 && s2) { return 1; } if(!s1 && !s2) { return 0; } return strcmp(s1, s2); } int main(int argc, char **argv) { if(argc < 2) { fprintf(stderr, "Missing command\n"); print_usage(argv[0]); return -1; } char *cmd = argv[1]; CmdArgs *args = cmd_parse_args(argc - 2, argv + 2); if(!args) { print_usage(argv[0]); return -1; } int ret = EXIT_FAILURE; if(!strcasecmp(cmd, "version") || !strcasecmp(cmd, "-version") || !strcasecmp(cmd, "--version")) { fprintf(stderr, "dav-sync %s\n", DAV_VERSION); cmd_args_free(args); return -1; } xmlGenericErrorFunc fnc = xmlerrorfnc; initGenericErrorDefaultFunc(&fnc); ctx = dav_context_new(); int cfgret = load_config(ctx) || load_sync_config(); pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_lock(&mutex); pthread_t tid; if(!strcmp(cmd, "check") || !strcmp(cmd, "check-config")) { if(!cfgret) { fprintf(stdout, "Configuration OK.\n"); ret = EXIT_SUCCESS; } else { /* no output, the warnings are written by load_config */ ret = EXIT_FAILURE; } } else if(!cfgret) { if(!strcmp(cmd, "pull")) { tid = start_sighandler(&mutex); ret = cmd_pull(args); stop_sighandler(&mutex, tid); } else if(!strcmp(cmd, "push")) { tid = start_sighandler(&mutex); ret = cmd_push(args, FALSE); stop_sighandler(&mutex, tid); } else if(!strcmp(cmd, "archive")) { tid = start_sighandler(&mutex); ret = cmd_push(args, TRUE); stop_sighandler(&mutex, tid); } else if(!strcmp(cmd, "restore")) { tid = start_sighandler(&mutex); ret = cmd_restore(args); stop_sighandler(&mutex, tid); } else if(!strcmp(cmd, "resolve-conflicts")) { ret = cmd_resolve_conflicts(args); } else if(!strcmp(cmd, "delete-conflicts")) { ret = cmd_delete_conflicts(args); } else if(!strcmp(cmd, "list-versions")) { ret = cmd_list_versions(args); } else if(!strcmp(cmd, "trash-info")) { ret = cmd_trash_info(args); } else if(!strcmp(cmd, "empty-trash")) { ret = cmd_empty_trash(args); } else if(!strcmp(cmd, "add-tag")) { ret = cmd_add_tag(args); } else if(!strcmp(cmd, "remove-tag")) { ret = cmd_remove_tag(args); } else if(!strcmp(cmd, "set-tags")) { ret = cmd_set_tags(args); } else if(!strcmp(cmd, "list-tags")) { ret = cmd_list_tags(args); } else if(!strcmp(cmd, "add-dir") || !strcmp(cmd, "add-directory")) { ret = cmd_add_directory(args); } else if(!strcmp(cmd, "list-dirs") || !strcmp(cmd, "list-directories")) { ret = cmd_list_dirs(); } else if(!strcmp(cmd, "check-repos") || !strcmp(cmd, "check-repositories")) { ret = cmd_check_repositories(); } else { print_usage(argv[0]); } } // cleanup cmd_args_free(args); dav_context_destroy(ctx); free_config(); free_sync_config(); curl_global_cleanup(); xmlCleanupParser(); return ret; } void print_usage(char *cmd) { fprintf(stderr, "Usage: %s command [options] arguments...\n\n", cmd); fprintf(stderr, "Commands:\n"); fprintf(stderr, " pull [-cldr] [-t <tags>] <directory>\n"); fprintf(stderr, " push [-cldrVRM] [-t <tags>] <directory>\n"); fprintf(stderr, " archive [-cldVRM] [-t <tags>] <directory>\n"); fprintf(stderr, " restore [-ldRM] [-s <directory>] [file...]\n"); fprintf(stderr, " resolve-conflicts <directory>\n"); fprintf(stderr, " delete-conflicts <directory>\n"); fprintf(stderr, " trash-info <directory>\n"); fprintf(stderr, " empty-trash <directory>\n"); fprintf(stderr, " add-tag [-s <syncdir>] <file> <tag>\n"); fprintf(stderr, " remove-tag [-s <syncdir>] <file> <tag>\n"); fprintf(stderr, " set-tags [-s <syncdir>] <file> [tags]\n"); fprintf(stderr, " list-tags [-s <syncdir>] <file>\n\n"); fprintf(stderr, "Options:\n"); fprintf(stderr, " -c Disable conflict detection\n"); fprintf(stderr, " -l Lock the repository before access\n"); fprintf(stderr, " -d Don't lock the repository\n"); fprintf(stderr, " -t <tags> " "Only sync files which have the specified tags\n"); fprintf(stderr, " -r " "Remove resources not matching the tag filter\n"); fprintf(stderr, " -V Enable versioning\n"); fprintf(stderr, " -R Restore removed files\n"); fprintf(stderr, " -M Restore modified files\n"); fprintf(stderr, " -v Verbose output (all commands)\n\n"); fprintf(stderr, "Config commands:\n"); fprintf(stderr, " add-directory\n"); fprintf(stderr, " list-directories\n"); fprintf(stderr, " check-config\n"); fprintf(stderr, " check-repositories\n\n"); } static void handlesig(int sig) { if(sync_shutdown) { exit(-1); } fprintf(stderr, "abort\n"); sync_shutdown = 1; } static void* sighandler(void *data) { signal(SIGTERM, handlesig); signal(SIGINT, handlesig); pthread_mutex_t *mutex = data; pthread_mutex_lock(mutex); // block thread return NULL; } pthread_t start_sighandler(pthread_mutex_t *mutex) { pthread_t tid; if(pthread_create(&tid, NULL, sighandler, mutex)) { perror("pthread_create"); exit(-1); } return tid; } void stop_sighandler(pthread_mutex_t *mutex, pthread_t tid) { pthread_mutex_unlock(mutex); void *data; pthread_join(tid, &data); } static int res_matches_filter(SyncDirectory *dir, char *res_path) { // trash filter if (dir->trash) { sstr_t rpath = sstr(util_concat_path(dir->path, res_path)); if (sstrprefix(rpath, sstr(dir->trash))) { free(rpath.ptr); return 1; } free(rpath.ptr); } // include/exclude filter UCX_FOREACH(inc, dir->include) { regex_t* pattern = (regex_t*) inc->data; if (regexec(pattern, res_path, 0, NULL, 0) == 0) { UCX_FOREACH(exc, dir->exclude) { regex_t* pattern = (regex_t*) exc->data; if (regexec(pattern, res_path, 0, NULL, 0) == 0) { return 1; } } return 0; } } return 1; } static int res_matches_tags(DavResource *res, SyncTagFilter *tagfilter) { if(!tagfilter || tagfilter->mode == DAV_SYNC_TAGFILTER_OFF) { return 1; } // NOTE: currently not implementable //int scope = res->iscollection ? // DAV_SYNC_TAGFILTER_SCOPE_COLLECTION // : DAV_SYNC_TAGFILTER_SCOPE_RESOURCE; //if((tagfilter->scope & scope) != scope) { // return 1; //} if(res->iscollection) { return 1; } DavXmlNode *tagsprop = dav_get_property_ns(res, DAV_NS, "tags"); UcxList *res_tags = parse_dav_xml_taglist(tagsprop); int ret = matches_tagfilter(res_tags, tagfilter); ucx_list_free_content(res_tags, (ucx_destructor) free_dav_tag); ucx_list_free(res_tags); return ret; } static int localres_matches_tags( SyncDirectory *dir, LocalResource *res, SyncTagFilter *tagfilter) { if(!tagfilter || tagfilter->mode == DAV_SYNC_TAGFILTER_OFF) { return 1; } //int scope = res->isdirectory ? // DAV_SYNC_TAGFILTER_SCOPE_COLLECTION // : DAV_SYNC_TAGFILTER_SCOPE_RESOURCE; //if((tagfilter->scope & scope) != scope) { // return 1; //} if(res->isdirectory) { return 1; } DavBool changed = 0; UcxList *res_tags = sync_get_file_tags(dir, res, &changed, NULL); int ret = matches_tagfilter(res_tags, tagfilter); UCX_FOREACH(elm, res_tags) { DavTag *t = elm->data; free_dav_tag(t); } ucx_list_free(res_tags); return ret; } static DavSession* create_session(DavContext *ctx, Repository *repo, char *collection) { int flags = get_repository_flags(repo); char *url = repo->url; DavBool find_collection = TRUE; if((flags & DAV_SESSION_DECRYPT_NAME) != DAV_SESSION_DECRYPT_NAME) { url = util_concat_path(repo->url, collection); find_collection = FALSE; } if(!collection || (collection[0] == '/' && strlen(collection) == 1)) { // collection is NULL or "/" // we don't need to find any collection because the repo url is // the base url find_collection = FALSE; } DavSession *sn = dav_session_new_auth( ctx, url, repo->user, repo->password); if(url != repo->url) { free(url); } sn->flags = flags; sn->key = dav_context_get_key(ctx, repo->default_key); curl_easy_setopt(sn->handle, CURLOPT_HTTPAUTH, repo->authmethods); curl_easy_setopt(sn->handle, CURLOPT_SSLVERSION, repo->ssl_version); if(repo->cert) { curl_easy_setopt(sn->handle, CURLOPT_CAPATH, repo->cert); } if(!repo->verification) { curl_easy_setopt(sn->handle, CURLOPT_SSL_VERIFYPEER, 0); curl_easy_setopt(sn->handle, CURLOPT_SSL_VERIFYHOST, 0); } if(find_collection) { DavResource *col = dav_resource_new(sn, collection); dav_exists(col); // exec this to get the href // we actually don't care what the result is // if it doesn't exists, an error will occur later // and we can't handle it here char *newurl = util_concat_path(repo->url, util_resource_name(col->href)); dav_session_set_baseurl(sn, newurl); free(newurl); } return sn; } static void print_allowed_cmds(SyncDirectory *dir) { fprintf(stderr, "Allowed commands: "); char *sep = ""; if((dir->allow_cmd & SYNC_CMD_PULL) == SYNC_CMD_PULL) { fprintf(stderr, "pull"); sep = ", "; } if((dir->allow_cmd & SYNC_CMD_PUSH) == SYNC_CMD_PUSH) { fprintf(stderr, "%spush", sep); sep = ", "; } if((dir->allow_cmd & SYNC_CMD_ARCHIVE) == SYNC_CMD_ARCHIVE) { fprintf(stderr, "%sarchive", sep); } fprintf(stderr, "\n"); } static void localres_keep(SyncDatabase *db, const char *path) { LocalResource *local = ucx_map_cstr_remove(db->resources, path); if(local) { local->keep = TRUE; } } void res2map(DavResource *root, UcxMap *map) { UcxList *stack = ucx_list_prepend(NULL, root->children); while(stack) { DavResource *res = stack->data; stack = ucx_list_remove(stack, stack); while(res) { ucx_map_cstr_put(map, res->path, res); if(res->children) { stack = ucx_list_prepend(stack, res->children); } res = res->next; } } } int cmd_pull(CmdArgs *a) { if(a->argc != 1) { fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many"); return -1; } // if there are syntax errors in the command line, fail asap. SyncTagFilter* tagfilter = parse_tagfilter_string( cmd_getoption(a, "tags"), DAV_SYNC_TAGFILTER_SCOPE_RESOURCE); if (!tagfilter) { fprintf(stderr, "Malformed tag filter\n"); return -1; } // TODO: tons of memory leaks... // call free_tagfilter() before each return SyncDirectory *dir = scfg_get_dir(a->argv[0]); if(!dir) { fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]); return -1; } if(scfg_check_dir(dir)) { return -1; } if((dir->allow_cmd & SYNC_CMD_PULL) != SYNC_CMD_PULL) { fprintf(stderr, "Command 'pull' is not allowed for this sync dir\n"); print_allowed_cmds(dir); return -1; } Repository *repo = get_repository(sstr(dir->repository)); if(!repo) { fprintf(stderr, "Unknown repository %s\n", dir->repository); return -1; } SyncDatabase *db = load_db(dir->database); if(!db) { fprintf(stderr, "Cannot load database file: %s\n", dir->database); return -1; } remove_deleted_conflicts(dir, db); DavSession *sn = create_session(ctx, repo, dir->collection); ucx_mempool_reg_destr(sn->mp, db, (ucx_destructor)destroy_db); if (cmd_getoption(a, "verbose")) { curl_easy_setopt(sn->handle, CURLOPT_VERBOSE, 1L); curl_easy_setopt(sn->handle, CURLOPT_STDERR, stderr); } // lock repository char *locktokenfile = NULL; DavBool locked = FALSE; DavResource *root = dav_resource_new(sn, "/"); root->iscollection = TRUE; if((dir->lockpush || cmd_getoption(a, "lock")) && !cmd_getoption(a, "nolock")) { if(dav_lock_t(root, dir->lock_timeout)) { print_resource_error(sn, "/"); dav_session_destroy(sn); fprintf(stderr, "Abort\n"); return -1; } DavLock *lock = dav_get_lock(sn, "/"); if(lock) { printf("Lock-Token: %s\n", lock->token); } locked = TRUE; locktokenfile = create_locktoken_file(dir->name, lock->token); } int ret = 0; DavResource *ls = dav_query(sn, "select D:getetag,idav:status,idav:tags,idav:finfo,idav:xattributes from / with depth = infinity"); if(!ls) { print_resource_error(sn, "/"); if(locked) { if(dav_unlock(root)) { print_resource_error(sn, "/"); } else { locked = FALSE; } } fprintf(stderr, "Abort\n"); dav_session_destroy(sn); // TODO: free return -1; } if(!ls->iscollection) { fprintf(stderr, "%s is not a collection.\nAbort.\n", ls->path); if(locked) { if(dav_unlock(root)) { print_resource_error(sn, "/"); } else { locked = FALSE; } } // TODO: free dav_session_destroy(sn); if(!locked && locktokenfile) { remove(locktokenfile); } return -1; } DavBool remove_file = cmd_getoption(a, "remove") ? 1 : 0; int sync_success = 0; int sync_delete = 0; int sync_error = 0; UcxList *res_modified = NULL; UcxList *res_new = NULL; UcxList *res_conflict = NULL; UcxList *res_mkdir = NULL; UcxList *res_metadata = NULL; UcxList *res_broken = NULL; UcxList *lres_removed = NULL; // list of LocalResource* //UcxMap *svrres = ucx_map_new(db->resources->count); UcxMap *dbres = ucx_map_clone(db->resources, NULL, NULL); UcxList *statls = NULL; UcxList *stack = ucx_list_prepend(NULL, ls->children); while(stack) { DavResource *res = stack->data; stack = ucx_list_remove(stack, stack); while(res) { DavBool res_filtered = FALSE; if (res_matches_filter(dir, res->path)) { res_filtered = TRUE; } else { UCX_FOREACH(elm, dir->tagfilter) { SyncTagFilter *tf = elm->data; if(!res_matches_tags(res, tf)) { res_filtered = TRUE; break; } } } if(res_filtered) { // don't delete files filtered by config localres_keep(db, res->path); res = res->next; continue; } if (!res_matches_tags(res, tagfilter)) { if(!remove_file) { localres_keep(db, res->path); } res = res->next; continue; } char *status = dav_get_string_property(res, "idav:status"); if(status && !strcmp(status, "broken")) { res = res->next; localres_keep(db, res->path); res_broken = ucx_list_append(res_broken, res); continue; } // check if a resource has changed on the server int change = resource_get_remote_change(a, res, dir, db); switch(change) { case REMOTE_NO_CHANGE: break; case REMOTE_CHANGE_MODIFIED: { res_modified = ucx_list_append(res_modified, res); break; } case REMOTE_CHANGE_NEW: { res_new = ucx_list_append(res_new, res); break; } case REMOTE_CHANGE_DELETED: break; // never happens case REMOTE_CHANGE_CONFLICT_LOCAL_MODIFIED: { res_conflict = ucx_list_append(res_conflict, res); break; } case REMOTE_CHANGE_METADATA: { res_metadata = ucx_list_append(res_metadata, res); break; } case REMOTE_CHANGE_MKDIR: { res_mkdir = ucx_list_append(res_mkdir, res); break; } } // remove every server resource from dbres // all remaining elements are the resources that are removed // on the server ucx_map_cstr_remove(dbres, res->path); if(res->children) { stack = ucx_list_prepend(stack, res->children); } res = res->next; } } // find deleted resources // svrres currently contains all resources from the server // and will replace the current db->resources map later UcxMapIterator i = ucx_map_iterator(dbres); LocalResource *local; UCX_MAP_FOREACH(key, local, i) { if (res_matches_filter(dir, local->path)) { continue; } if(!local->keep) { lres_removed = ucx_list_prepend(lres_removed, local); } } // the first thing we need are all directories to put the files in UCX_FOREACH(elm, res_mkdir) { DavResource *res = elm->data; char *local_path = util_concat_path(dir->path, res->path); if(sys_mkdir(local_path) && errno != EEXIST) { fprintf(stderr, "Cannot create directory %s: %s", local_path, strerror(errno)); } free(local_path); } // we need a map for all conflicts for fast lookups UcxMap *conflicts = ucx_map_new(ucx_list_size(res_conflict)+16); UCX_FOREACH(elm, res_conflict) { DavResource *res = elm->data; ucx_map_cstr_put(conflicts, res->path, res); } // download all new, modified and conflict files UcxList *download = ucx_list_concat(res_modified, res_conflict); download = ucx_list_concat(res_new, download); UCX_FOREACH(elm, download) { DavResource *res = elm->data; if(sync_shutdown) { break; } if(ucx_map_cstr_get(conflicts, res->path)) { rename_conflict_file(dir, db, res->path); } // download the resource if(sync_get_resource(a, dir, res, db, &sync_success)) { fprintf(stderr, "resource download failed: %s\n", res->path); sync_error++; } } UCX_FOREACH(elm, res_metadata) { DavResource *res = elm->data; if(sync_shutdown) { break; } LocalResource *local = ucx_map_cstr_get(db->resources, res->path); if(local) { printf("update: %s\n", res->path); char *local_path = util_concat_path(dir->path, res->path); if(sync_store_metadata(dir, local_path, local, res)) { fprintf(stderr, "Metadata update failed: %s\n", res->path); sync_error++; } else { struct stat s; if(stat(local_path, &s)) { fprintf(stderr, "Cannot stat file after update: %s\n", strerror(errno)); } sync_set_metadata_from_stat(local, &s); sync_success++; } free(local_path); } else { // this should never happen but who knows fprintf(stderr, "Cannot update metadata of file %s: not in database\n", res->path); } } UcxList *rmdirs = NULL; UCX_FOREACH(elm, lres_removed) { LocalResource *res = elm->data; if(sync_shutdown) { break; } int ret = sync_remove_local_resource(dir, res); if(ret == -1) { rmdirs = ucx_list_append(rmdirs, res); } else if(ret == 0) { LocalResource *local = ucx_map_cstr_remove(db->resources, res->path); if(local) { local_resource_free(local); } sync_delete++; } } UCX_FOREACH(elm, rmdirs) { LocalResource *local_dir = elm->data; if(!sync_remove_local_directory(dir, local_dir)) { LocalResource *local = ucx_map_cstr_remove(db->resources, local_dir->path); if(local) { local_resource_free(local); } sync_delete++; } } // unlock repository if(locked) { if(dav_unlock(root)) { print_resource_error(sn, "/"); ret = -1; } else { locked = FALSE; } } // store db if(store_db(db, dir->database, dir->db_settings)) { fprintf(stderr, "Cannot store sync db\n"); ret = -2; } // cleanup dav_session_destroy(sn); if(!locked && locktokenfile) { remove(locktokenfile); } // Report if(ret != -2) { char *str_success = sync_success == 1 ? "file" : "files"; char *str_delete = sync_delete == 1 ? "file" : "files"; char *str_error = sync_error == 1 ? "error" : "errors"; printf("Result: %d %s pulled, %d %s deleted, %d %s\n", sync_success, str_success, sync_delete,str_delete, sync_error, str_error); } return ret; } RemoteChangeType resource_get_remote_change( CmdArgs *a, DavResource *res, SyncDirectory *dir, SyncDatabase *db) { char *etag = dav_get_string_property(res, "D:getetag"); if(!etag) { fprintf(stderr, "Error: resource %s has no etag\n", res->path); return REMOTE_NO_CHANGE; } RemoteChangeType type = cmd_getoption(a, "conflict") ? REMOTE_CHANGE_MODIFIED : REMOTE_CHANGE_CONFLICT_LOCAL_MODIFIED; LocalResource *local = ucx_map_cstr_get(db->resources, res->path); char *local_path = util_concat_path(dir->path, res->path); SYS_STAT s; DavBool exists = 1; if(sys_stat(local_path, &s)) { if(errno != ENOENT) { fprintf(stderr, "Cannot stat file: %s\n", local_path); free(local_path); return REMOTE_NO_CHANGE; } exists = 0; } RemoteChangeType ret = REMOTE_NO_CHANGE; if(res->iscollection) { if(!exists) { ret = REMOTE_CHANGE_MKDIR; } } else if(local) { DavBool nochange = FALSE; if(local->etag) { sstr_t e = sstr(etag); if(sstrprefix(e, S("W/"))) { e = sstrsubs(e, 2); } if(!strcmp(e.ptr, local->etag)) { // resource is already up-to-date on the client nochange = TRUE; } } if(!nochange) { if(!(exists && s.st_mtime != local->last_modified)) { type = REMOTE_CHANGE_MODIFIED; } ret = type; } } else if(exists) { ret = type; } else { ret = REMOTE_CHANGE_NEW; } while(ret == REMOTE_NO_CHANGE && local) { // check if tags have changed if(dir->tagconfig) { DavXmlNode *tagsprop = dav_get_property_ns(res, DAV_NS, "tags"); UcxList *remote_tags = NULL; if(tagsprop) { remote_tags = parse_dav_xml_taglist(tagsprop); } char *remote_hash = create_tags_hash(remote_tags); if(nullstrcmp(remote_hash, local->remote_tags_hash)) { ret = REMOTE_CHANGE_METADATA; } if(remote_hash) { free(remote_hash); } free_taglist(remote_tags); if(ret == REMOTE_CHANGE_METADATA) { break; } } // check if extended attributes have changed if(dir->metadata & FINFO_XATTR == FINFO_XATTR) { DavXmlNode *xattr = dav_get_property_ns(res, DAV_NS, "xattributes"); char *xattr_hash = get_xattr_hash(xattr); if(nullstrcmp(xattr_hash, local->xattr_hash)) { ret = REMOTE_CHANGE_METADATA; break; } } // check if finfo has changed DavXmlNode *finfo = dav_get_property_ns(res, DAV_NS, "finfo"); if(dir->metadata & FINFO_MODE == FINFO_MODE) { FileInfo f; finfo_get_values(finfo, &f); if(f.mode_set && f.mode != local->mode) { ret = REMOTE_CHANGE_METADATA; break; } } break; } free(local_path); return ret; } void sync_set_metadata_from_stat(LocalResource *local, struct stat *s) { local->last_modified = s->st_mtime; local->mode = s->st_mode & 07777; local->uid = s->st_uid; local->gid = s->st_gid; local->size = s->st_size; } int sync_get_resource( CmdArgs *a, SyncDirectory *dir, DavResource *res, SyncDatabase *db, int *counter) { LocalResource *local = ucx_map_cstr_get(db->resources, res->path); char *local_path = util_concat_path(dir->path, res->path); char *etag = dav_get_string_property(res, "D:getetag"); SYS_STAT s; memset(&s, 0, sizeof(SYS_STAT)); int ret = 0; char *tmp_path = create_tmp_download_path(local_path); if(!tmp_path) { fprintf(stderr, "Cannot create tmp path for %s\n", local_path); free(local_path); return -1; } FILE *out = sys_fopen(tmp_path, "wb"); if(!out) { fprintf(stderr, "Cannot open output file: %s\n", local_path); free(local_path); free(tmp_path); return -1; } printf("get: %s\n", res->path); if(dav_get_content(res, out, (dav_write_func)fwrite)) { ret = -1; } fclose(out); if(ret == 0) { (*counter)++; if(sync_store_metadata(dir, tmp_path, local, res)) { fprintf(stderr, "Cannot store metadata: %s\n", res->path); } if(dir->trash && dir->backuppull) { move_to_trash(dir, local_path); } if(sys_rename(tmp_path, local_path)) { fprintf( stderr, "Cannot rename file %s to %s\n", tmp_path, local_path); perror(""); free(tmp_path); free(local_path); return -1; } if(sys_stat(local_path, &s)) { fprintf(stderr, "Cannot stat file %s: %s\n", local_path, strerror(errno)); } if(!local) { // new local resource local = calloc(1, sizeof(LocalResource)); local->path = strdup(res->path); ucx_map_cstr_put(db->resources, local->path, local); } if(local->etag) { free(local->etag); } // set metadata from stat local->etag = strdup(etag); sync_set_metadata_from_stat(local, &s); local->skipped = FALSE; } else { if(sys_unlink(tmp_path)) { fprintf(stderr, "Cannot remove tmp file: %s\n", tmp_path); } } free(tmp_path); free(local_path); return ret; } int sync_remove_local_resource(SyncDirectory *dir, LocalResource *res) { char *local_path = util_concat_path(dir->path, res->path); SYS_STAT s; if(sys_stat(local_path, &s)) { free(local_path); return -2; } if(S_ISDIR(s.st_mode)) { free(local_path); return -1; } if(s.st_mtime != res->last_modified) { free(local_path); return -2; } printf("delete: %s\n", res->path); int ret = 0; if(dir->trash) { move_to_trash(dir, local_path); } else if(sys_unlink(local_path)) { fprintf(stderr, "Cannot remove file %s\n", local_path); ret = -2; } free(local_path); return ret; } int sync_remove_local_directory(SyncDirectory *dir, LocalResource *res) { int ret = 0; char *local_path = util_concat_path(dir->path, res->path); printf("delete: %s\n", res->path); if(rmdir(local_path)) { fprintf(stderr, "rmdir: %s : %s", local_path, strerror(errno)); ret = 1; } free(local_path); return ret; } void rename_conflict_file(SyncDirectory *dir, SyncDatabase *db, char *path) { char *local_path = util_concat_path(dir->path, path); char *parent = util_parent_path(local_path); int rev = 0; SYS_STAT s; int loop = 1; do { char *res_parent = util_parent_path(path); char *res_name = util_resource_name(path); sstr_t new_path = ucx_sprintf( "%sorig.%d.%s", parent, rev, res_name); sstr_t new_res_path = ucx_sprintf( "%sorig.%d.%s", res_parent, rev, res_name); if(sys_stat(new_path.ptr, &s)) { if(errno == ENOENT) { loop = 0; printf("conflict: %s\n", local_path); if(sys_rename(local_path, new_path.ptr)) { //printf("errno: %d\n", errno); fprintf( stderr, "Cannot rename file %s to %s\n", local_path, new_path.ptr); } else { LocalResource *conflict = calloc(1, sizeof(LocalResource)); conflict->path = strdup(new_res_path.ptr); ucx_map_cstr_put(db->conflict, new_res_path.ptr, conflict); } } } rev++; free(res_parent); free(new_path.ptr); free(new_res_path.ptr); } while(loop); free(parent); free(local_path); } char* create_tmp_download_path(char *path) { char *new_path = NULL; char *parent = util_parent_path(path); for (int i=0;;i++) { sstr_t np = ucx_asprintf( ucx_default_allocator(), "%sdownload%d-%s", parent, i, util_resource_name(path)); SYS_STAT s; if(sys_stat(np.ptr, &s)) { if(errno == ENOENT) { new_path = np.ptr; } break; } free(np.ptr); }; free(parent); return new_path; } void move_to_trash(SyncDirectory *dir, char *path) { char *new_path = NULL; for (int i=0;;i++) { sstr_t np = ucx_asprintf( ucx_default_allocator(), "%s%d-%s", dir->trash, i, util_resource_name(path)); SYS_STAT s; if(sys_stat(np.ptr, &s)) { if(errno == ENOENT) { new_path = np.ptr; } break; } free(np.ptr); }; if(!new_path) { fprintf(stderr, "Cannot move file %s to trash.\n", path); return; } if(sys_rename(path, new_path)) { //printf("errno: %d\n", errno); fprintf( stderr, "Cannot rename file %s to %s\n", path, new_path); } free(new_path); } static int res_isconflict(SyncDatabase *db, LocalResource *res) { return ucx_map_cstr_get(db->conflict, res->path) ? 1 : 0; } int cmd_push(CmdArgs *a, DavBool archive) { if(a->argc != 1) { fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many"); return -1; } // if there are syntax errors in the command line, fail asap. SyncTagFilter* tagfilter = parse_tagfilter_string( cmd_getoption(a, "tags"), DAV_SYNC_TAGFILTER_SCOPE_RESOURCE); if (!tagfilter) { fprintf(stderr, "Malformed tag filter\n"); return -1; } SyncDirectory *dir = scfg_get_dir(a->argv[0]); if(!dir) { fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]); return -1; } if(scfg_check_dir(dir)) { return -1; } if(cmd_getoption(a, "versioning")) { if(dir->versioning) { dir->versioning->always = TRUE; } else { fprintf(stderr, "Error: versioning not configured for the sync directory\nAbort.\n"); return -1; } } int cmd = archive ? SYNC_CMD_ARCHIVE : SYNC_CMD_PUSH; if((dir->allow_cmd & cmd) != cmd) { fprintf(stderr, "Command '%s' is not allowed for this sync dir\n", archive ? "archive" : "push"); print_allowed_cmds(dir); return -1; } Repository *repo = get_repository(sstr(dir->repository)); if(!repo) { fprintf(stderr, "Unkown repository %s\n", dir->name); return -1; } SyncDatabase *db = load_db(dir->database); if(!db) { fprintf(stderr, "Cannot load database file: %s\n", dir->database); return -1; } remove_deleted_conflicts(dir, db); DavSession *sn = create_session(ctx, repo, dir->collection); ucx_mempool_reg_destr(sn->mp, db, (ucx_destructor)destroy_db); if (cmd_getoption(a, "verbose")) { curl_easy_setopt(sn->handle, CURLOPT_VERBOSE, 1L); curl_easy_setopt(sn->handle, CURLOPT_STDERR, stderr); } DavBool restore_removed = cmd_getoption(a, "restore-removed") ? 1 : 0; DavBool restore_modified = cmd_getoption(a, "restore-modified") ? 1 : 0; DavBool restore = restore_removed || restore_modified; int depth = restore ? -1 : 0; DavResource *root = dav_query(sn, "select D:getetag,idav:status from / with depth = %d", depth); if(!root) { print_resource_error(sn, "/"); dav_session_destroy(sn); fprintf(stderr, "Abort\n"); return -1; } UcxMap *svrres = NULL; if(restore) { svrres = ucx_map_new(1024); res2map(root, svrres); } int cdt = cmd_getoption(a, "conflict") ? 0 : 1; // conflict detection // lock repository DavBool locked = FALSE; char *locktokenfile = NULL; if((dir->lockpush || cmd_getoption(a, "lock")) && !cmd_getoption(a, "nolock")) { if(dav_lock_t(root, dir->lock_timeout)) { print_resource_error(sn, "/"); dav_session_destroy(sn); fprintf(stderr, "Abort\n"); return -1; } DavLock *lock = dav_get_lock(sn, "/"); if(lock) { printf("Lock-Token: %s\n", lock->token); } locked = TRUE; locktokenfile = create_locktoken_file(dir->name, lock->token); } DavBool remove_file = cmd_getoption(a, "remove") ? 1 : 0; int sync_success = 0; int sync_delete = 0; int sync_skipped = 0; int sync_error = 0; UcxList *ls_put = NULL; UcxList *ls_conflict = NULL; UcxList *ls_update = NULL; UcxList *ls_delete = NULL; // upload all changed files //UcxList *resources = cmd_getoption(a, "read") ? // read_changes(dir, db) : local_scan(dir, db); UcxList *resources = local_scan(dir, db); UcxMap *resources_map = ucx_map_new(ucx_list_size(resources)+16); UCX_FOREACH(elm, resources) { LocalResource *local_res = elm->data; // ignore all files, that are excluded by a static filter (sync.xml) // static include/exclude filter if(res_matches_filter(dir, local_res->path+1)) { continue; } // static tag filter UCX_FOREACH(elm, dir->tagfilter) { SyncTagFilter *tf = elm->data; if(!localres_matches_tags(dir, local_res, tf)) { continue; } } // we need a fast file lookup map later ucx_map_cstr_put(resources_map, local_res->path, local_res); // dynamic tag filter if(!localres_matches_tags(dir, local_res, tagfilter)) { if(!remove_file) { LocalResource *dbres = ucx_map_cstr_get( db->resources, local_res->path); if(dbres) { // this makes sure the file will not be deleted later dbres->keep = TRUE; } } continue; } if(res_isconflict(db, local_res)) { ls_conflict = ucx_list_append(ls_conflict, local_res); continue; } int is_changed = local_resource_is_changed( dir, db, local_res, svrres, restore_removed, restore_modified); if(is_changed) { ls_put = ucx_list_append(ls_put, local_res); } else if(local_res->metadata_updated) { ls_update = ucx_list_append(ls_update, local_res); } } // find all deleted files and cleanup the database UcxMapIterator i = ucx_map_iterator(db->resources); LocalResource *local; UcxList *removed_res = NULL; UCX_MAP_FOREACH(key, local, i) { // all filtered files should be removed from the database if(res_matches_filter(dir, local->path+1)) { ucx_map_cstr_remove(db->resources, local->path); continue; } UCX_FOREACH(elm, dir->tagfilter) { SyncTagFilter *tf = elm->data; if(!localres_matches_tags(dir, local, tf)) { ucx_map_cstr_remove(db->resources, local->path); continue; } } if(!ucx_map_get(resources_map, key)) { // The current LocalResource is in the database but doesn't exist // in the filesystem anymore. This means the file was deleted // and should be deleted on server if(!archive) { ls_delete = ucx_list_append(ls_delete, local); } else { removed_res = ucx_list_prepend(removed_res, local); } } } UCX_FOREACH(elm, removed_res) { LocalResource *local = elm->data; ucx_map_cstr_remove(db->resources, local->path); } ls_delete = ucx_list_sort(ls_delete, (cmp_func)resource_pathlen_cmp, NULL); // // BEGIN PUSH // // upload changed files int ret = 0; int error = 0; for(UcxList *elm=ls_put;elm && !sync_shutdown;elm=elm->next) { LocalResource *local_res = elm->data; DavResource *res = dav_resource_new(sn, local_res->path); if(!res) { print_resource_error(sn, local_res->path); ret = -1; sync_error++; } if(local_res->isdirectory) { dav_exists(res); if(sn->error == DAV_NOT_FOUND) { int abort = 0; // make sure to store tags for newly created cols local_res->tags_updated = 1; // create collection // TODO: show 405 printf("mkcol: %s\n", local_res->path); if(sync_mkdir(dir, res, local_res) && sn->error != DAV_METHOD_NOT_ALLOWED) { print_resource_error(sn, res->path); ret = -1; sync_error++; error = 1; abort = 1; } if(local_res->metadata_updated && !abort) { sync_update_metadata(dir, sn, res, local_res); } } else if(sn->error != DAV_OK) { // dav_exists() failed print_resource_error(sn, local_res->path); ret = -1; sync_error++; error = 1; } } else { if(cdt && remote_resource_is_changed(sn, dir, db, res, local_res)) { printf("conflict: %s\n", local_res->path); local_res->last_modified = 0; local_res->skipped = TRUE; sync_skipped++; } else { printf("put: %s\n", local_res->path); if(sync_put_resource(dir, res, local_res, &sync_success)) { sync_error++; print_resource_error(sn, res->path); ret = -1; error = 1; } } } dav_resource_free(res); LocalResource *dbres = ucx_map_cstr_remove(db->resources, local_res->path); ucx_map_cstr_put(db->resources, local_res->path, local_res); //if(dbres) local_resource_free(dbres); } // metadata updates for(UcxList *elm=ls_update;elm && !sync_shutdown;elm=elm->next) { LocalResource *local_res = elm->data; DavResource *res = dav_resource_new(sn, local_res->path); if(dir->tagconfig) { DavPropName properties[] = { {DAV_NS,"tags"}, }; if(dav_load_prop(res, properties, 1)) { sync_error++; print_resource_error(sn, res->path); ret = -1; error = 1; continue; } } if(local_res->metadata_updated) { if(!sync_update_metadata(dir, sn, res, local_res)) { LocalResource *dbres = ucx_map_cstr_remove(db->resources, local_res->path); ucx_map_cstr_put(db->resources, local_res->path, local_res); } } } // delete all removed files UcxList *cols = NULL; UcxList **col_list = &cols; UcxList *deletelist = ls_delete; for(int i=0;i<2;i++) { // the first iteration deletes everything from ls_delete except // all collections, which are stored in cols // in the second run all collections will be deleted for(UcxList *elm=deletelist;elm && !sync_shutdown;elm=elm->next) { LocalResource *local = elm->data; if(local->keep) { continue; } if(sync_delete_remote_resource(dir, sn, local, &sync_delete, col_list)) { if(sn->error != DAV_NOT_FOUND) { print_resource_error(sn, local->path); sync_error++; break; } } else { LocalResource *dbres = ucx_map_cstr_remove(db->resources, local->path); //local_resource_free(dbres); } } deletelist = cols; col_list = NULL; } // unlock repository if(locked) { if(dav_unlock(root)) { print_resource_error(sn, "/"); ret = -1; } else { locked = FALSE; } } // store db if(store_db(db, dir->database, dir->db_settings)) { fprintf(stderr, "Cannot store sync db\n"); ret = -2; } // cleanup if(!locked && locktokenfile) { remove(locktokenfile); } //ucx_map_free_content(db->resources, (ucx_destructor)local_resource_free); //ucx_map_free(db->resources); dav_session_destroy(sn); // Report if(ret != -2) { char *str_success = sync_success == 1 ? "file" : "files"; char *str_delete = sync_delete == 1 ? "file" : "files"; char *str_skipped = sync_delete == 1 ? "file" : "files"; char *str_error = sync_error == 1 ? "error" : "errors"; printf("Result: %d %s pushed, ", sync_success, str_success); if(!archive) { printf("%d %s deleted, ", sync_delete, str_delete); } printf("%d %s skipped, %d %s\n", sync_skipped, str_skipped, sync_error, str_error); } return ret; } static int localres_cmp_path(LocalResource *a, LocalResource *b, void *n) { return strcmp(a->path, b->path); } int cmd_restore(CmdArgs *a) { char *syncdir = cmd_getoption(a, "syncdir"); if(!syncdir && a->argc == 0) { fprintf(stderr, "No syncdir or files specified\n"); return -1; } SyncDirectory *dir = NULL; UcxMap *files = NULL; if(syncdir) { dir = scfg_get_dir(syncdir); } LocalResource nres; if(a->argc > 0) { files = ucx_map_new(a->argc+8); // get all specified files and check the syncdir SyncDirectory *sd = NULL; for(int i=0;i<a->argc;i++) { SyncFile f; int err = sync_get_file(a, a->argv[i], &f, FALSE); if(err) { sync_print_get_file_err(a->argv[i], err); return 1; } if(!sd) { sd = f.dir; } else { if(f.dir != sd) { fprintf(stderr, "Not all files are in the same syncdir\n"); return 1; } } ucx_map_cstr_put(files, f.path, &nres); } dir = sd; } if(!dir) { fprintf(stderr, "Unknown sync dir: %s\n", syncdir); return -1; } if(scfg_check_dir(dir)) { return -1; } if((dir->allow_cmd & SYNC_CMD_RESTORE) != SYNC_CMD_RESTORE) { fprintf(stderr, "Command ''restore'' is not allowed for this sync dir\n"); print_allowed_cmds(dir); return -1; } DavBool restore_modified = cmd_getoption(a, "restore-modified") ? 1 : 0; DavBool restore_removed = cmd_getoption(a, "restore-removed") ? 1 : 0; if(!restore_modified && !restore_removed) { restore_modified = 1; restore_removed = 1; } SyncDatabase *db = load_db(dir->database); if(!db) { fprintf(stderr, "Cannot load database file: %s\n", dir->database); return -1; } remove_deleted_conflicts(dir, db); UcxList *modified = NULL; UcxList *deleted = NULL; // iterate over all db resources and check if any resource is // modified or deleted UcxMapIterator i = ucx_map_iterator(files ? files : db->resources); LocalResource *resource; UCX_MAP_FOREACH(key, resource, i) { if(resource == &nres) { resource = ucx_map_get(db->resources, key); } char *file_path = util_concat_path(dir->path, resource->path); SYS_STAT s; if(sys_stat(file_path, &s)) { if(errno == ENOENT) { if(restore_removed) { deleted = ucx_list_prepend(deleted, resource); } } else { fprintf(stderr, "Cannot stat file: %s\n", file_path); perror(""); } } else if(!resource->isdirectory && !S_ISDIR(s.st_mode)) { if(resource->last_modified != s.st_mtime || resource->size != s.st_size) { if(restore_modified) { modified = ucx_list_prepend(modified, resource); } } } free(file_path); } if(files) { ucx_map_free(files); } int ret = 0; // create DavSession Repository *repo = get_repository(sstr(dir->repository)); if(!repo) { fprintf(stderr, "Unkown repository %s\n", dir->name); return -1; } DavSession *sn = create_session(ctx, repo, dir->collection); ucx_mempool_reg_destr(sn->mp, db, (ucx_destructor)destroy_db); if (cmd_getoption(a, "verbose")) { curl_easy_setopt(sn->handle, CURLOPT_VERBOSE, 1L); curl_easy_setopt(sn->handle, CURLOPT_STDERR, stderr); } // lock repository char *locktokenfile = NULL; DavBool locked = FALSE; DavResource *root = dav_resource_new(sn, "/"); root->iscollection = TRUE; if((dir->lockpush || cmd_getoption(a, "lock")) && !cmd_getoption(a, "nolock")) { if(dav_lock_t(root, dir->lock_timeout)) { print_resource_error(sn, "/"); dav_session_destroy(sn); fprintf(stderr, "Abort\n"); return -1; } DavLock *lock = dav_get_lock(sn, "/"); if(lock) { printf("Lock-Token: %s\n", lock->token); } locked = TRUE; locktokenfile = create_locktoken_file(dir->name, lock->token); } int sync_success = 0; int sync_error = 0; UcxList *resources = ucx_list_concat(modified, deleted); resources = ucx_list_sort(resources, (cmp_func)localres_cmp_path, NULL); UCX_FOREACH(elm, resources) { LocalResource *resource = elm->data; DavResource *res = dav_get(sn, resource->path, "D:getetag,idav:status,idav:finfo,idav:xattributes"); char *status = dav_get_string_property(res, "idav:status"); if(status && !strcmp(status, "broken")) { continue; } // download the resource if(!sync_shutdown && sync_get_resource(a, dir, res, db, &sync_success)) { fprintf(stderr, "sync_get_resource failed for resource: %s\n", res->path); sync_error++; } } // unlock repository if(locked) { if(dav_unlock(root)) { print_resource_error(sn, "/"); ret = -1; } else { locked = FALSE; } } // store db if(store_db(db, dir->database, dir->db_settings)) { fprintf(stderr, "Cannot store sync db\n"); ret = -2; } // cleanup dav_session_destroy(sn); if(!locked && locktokenfile) { remove(locktokenfile); } // Report if(ret != -2) { char *str_success = sync_success == 1 ? "file" : "files"; char *str_error = sync_error == 1 ? "error" : "errors"; printf("Result: %d %s pulled, %d %s\n", sync_success, str_success, sync_error, str_error); } return 0; } UcxList* local_scan(SyncDirectory *dir, SyncDatabase *db) { UcxList *resources = NULL; char *path = strdup("/"); UcxList *stack = ucx_list_prepend(NULL, path); while(stack) { // get a directory path from the stack and read all entries // if an entry is a directory, put it on the stack char *p = stack->data; stack = ucx_list_remove(stack, stack); char *local_path = util_concat_path(dir->path, p); SYS_DIR local_dir = sys_opendir(local_path); if(!local_dir) { fprintf(stderr, "Cannot open directory %s\n", local_path); } else { SysDirEnt *ent; while((ent = sys_readdir(local_dir)) != NULL) { if(!strcmp(ent->name, ".") || !strcmp(ent->name, "..")) { continue; } char *new_path = util_concat_path(p, ent->name); int isdir = 0; LocalResource *res = local_resource_new(dir, db, new_path, &isdir); if(isdir) { resources = ucx_list_append(resources, res); stack = ucx_list_prepend(stack, new_path); } else if(res) { resources = ucx_list_append(resources, res); free(new_path); } else { free(new_path); } } sys_closedir(local_dir); } free(local_path); free(p); } return resources; } UcxList* read_changes(SyncDirectory *dir, SyncDatabase *db) { UcxProperties *parser = ucx_properties_new(); parser->delimiter = ':'; UcxList *resources = NULL; sstr_t name; sstr_t value; char buf[STDIN_BUF_SIZE]; size_t r; while(!feof(stdin)) { r = fread(buf, 1, STDIN_BUF_SIZE, stdin); ucx_properties_fill(parser, buf, r); while(ucx_properties_next(parser, &name, &value)) { if(value.length == 0) { fprintf(stderr, "Wrong input\n"); continue; } if(value.ptr[0] == '"' && value.length > 2 && value.ptr[value.length - 1] == '"') { value.ptr[value.length - 1] = '\0'; value.ptr++; value.length -= 2; } value = sstrdup(value); if(!sstrcmp(name, S("put"))) { int isdir; LocalResource *res = local_resource_new(dir, db, value.ptr, &isdir); if(res) { resources = ucx_list_append(resources, res); } } else if(!sstrcmp(name, S("remove"))) { ucx_map_sstr_remove(db->resources, value); } free(value.ptr); } } ucx_properties_free(parser); return resources; } LocalResource* local_resource_new(SyncDirectory *dir, SyncDatabase *db, char *path, int *isdir) { char *file_path = util_concat_path(dir->path, path); SYS_STAT s; if(sys_stat(file_path, &s)) { fprintf(stderr, "Cannot stat file %s\n", file_path); free(file_path); return NULL; } free(file_path); if(!S_ISDIR(s.st_mode)) { *isdir = 0; LocalResource *res = calloc(1, sizeof(LocalResource)); res->path = strdup(path); res->etag = NULL; res->last_modified = s.st_mtime; res->size = s.st_size; res->mode = s.st_mode & 07777; res->uid = s.st_uid; res->gid = s.st_gid; return res; } else { *isdir = 1; LocalResource *res = calloc(1, sizeof(LocalResource)); res->path = util_concat_path(path, "/"); res->last_modified = s.st_mtime; res->mode = s.st_mode & 07777; res->uid = s.st_uid; res->gid = s.st_gid; res->isdirectory = 1; return res; } } LocalResource* local_resource_copy(LocalResource *res) { LocalResource *newres = calloc(1, sizeof(LocalResource)); if(res->name) { newres->name = strdup(res->name); } if(res->path) { newres->path = strdup(res->path); } if(res->etag) { newres->etag = strdup(res->etag); } newres->skipped = res->skipped; newres->size = res->size; newres->last_modified = res->last_modified; newres->isdirectory = res->isdirectory; if(res->tags_hash) { newres->tags_hash = strdup(res->tags_hash); } newres->tags_updated = res->tags_updated; return newres; } int local_resource_is_changed( SyncDirectory *dir, SyncDatabase *db, LocalResource *res, UcxMap *svrres, DavBool restore_removed, DavBool restore_modified) { LocalResource *db_res = ucx_map_cstr_get(db->resources, res->path); res->tags_updated = 0; if(db_res) { if(svrres) { DavResource *remote = ucx_map_cstr_get(svrres, res->path); if(restore_removed && !remote) { return 1; } if(!res->isdirectory && restore_modified && remote) { char *etag = dav_get_string_property(remote, "D:getetag"); if(!etag || (db_res->etag && strcmp(etag, db_res->etag))) { res->restore = TRUE; return 1; } } } res->tags_updated = db_res->tags_updated; if(db_res->etag) { res->etag = strdup(db_res->etag); } if(db_res->tags_hash) { res->tags_hash = strdup(db_res->tags_hash); } if(db_res->remote_tags_hash) { res->remote_tags_hash = strdup(db_res->remote_tags_hash); } if(db_res->xattr_hash) { res->xattr_hash = strdup(db_res->xattr_hash); } if(dir->tagconfig && dir->tagconfig->detect_changes && !res->tags_updated) { UcxBuffer *tags = sync_get_file_tag_data(dir, res); if(tags) { if(db_res->tags_hash) { char *hash = dav_create_hash(tags->space, tags->size); if(strcmp(hash, db_res->tags_hash)) { res->tags_updated = 1; } free(hash); } else { res->tags_updated = 1; } } else if(db_res->tags_hash) { res->tags_updated = 1; // tags removed } res->metadata_updated = res->tags_updated; } if(dir->metadata & FINFO_MODE == FINFO_MODE) { if(db_res->mode != res->mode) { res->finfo_updated = 1; res->metadata_updated = 1; } } if(dir->metadata & FINFO_OWNER == FINFO_OWNER) { if(db_res->uid != res->uid || db_res->gid != res->gid) { res->finfo_updated = 1; res->metadata_updated = 1; } } if(dir->metadata & FINFO_XATTR == FINFO_XATTR) { char *path = util_concat_path(dir->path, db_res->path); XAttributes *xattr = file_get_attributes(path); // test if xattr are added, removed or changed if((db_res->xattr_hash && !xattr) || (!db_res->xattr_hash && xattr) || (xattr && db_res->xattr_hash && strcmp(xattr->hash, db_res->xattr_hash))) { res->metadata_updated = 1; res->xattr_updated = 1; res->xattr = xattr; } else if(xattr) { xattributes_free(xattr); } } if(db_res->last_modified == res->last_modified && db_res->size == res->size) { return 0; } } else { res->tags_updated = 1; res->finfo_updated = 1; res->xattr_updated = 1; res->metadata_updated = 1; } return 1; } int remote_resource_is_changed( DavSession *sn, SyncDirectory *dir, SyncDatabase *db, DavResource *remote, LocalResource *res) { DavPropName properties[] = { {"DAV:","getetag"}, {DAV_NS,"tags"}, {DAV_NS,VERSION_PATH_PROPERTY} }; int err = dav_load_prop(remote, properties, 3); if(res->restore) { return 0; } int ret = 0; if(err == 0) { char *etag = dav_get_string_property(remote, "D:getetag"); if(!res->etag) { // the resource is on the server and the client has no etag ret = 1; } else if(etag) { sstr_t e = sstr(etag); if(sstrprefix(e, S("W/"))) { e = sstrsubs(e, 2); } if(strcmp(e.ptr, res->etag)) { ret = 1; } } else { // something weird is happening, the server must support etags fprintf(stderr, "Warning: resource %s has no etag\n", remote->href); } } return ret; } int resource_pathlen_cmp(LocalResource *res1, LocalResource *res2, void *n) { size_t s1 = strlen(res1->path); size_t s2 = strlen(res2->path); if(s1 < s2) { return 1; } else if(s1 > s2) { return -1; } else { return 0; } } int sync_set_status(DavResource *res, char *status) { DavResource *resource = dav_resource_new(res->session, res->path); dav_set_string_property(resource, "idav:status", status); int ret = dav_store(resource); dav_resource_free(resource); return ret; } int sync_remove_status(DavResource *res) { DavResource *resource = dav_resource_new(res->session, res->path); dav_remove_property(resource, "idav:status"); int ret = dav_store(resource); dav_resource_free(resource); return ret; } int sync_tags_equal(UcxList *tags1, UcxList *tags2) { if(!tags1) { return tags2 ? 0 : 1; } if(!tags2) { return tags1 ? 0 : 1; } UcxMap *map1 = ucx_map_new(32); UCX_FOREACH(elm, tags1) { DavTag *t = elm->data; ucx_map_cstr_put(map1, t->name, t); } int equal = 1; int i = 0; UCX_FOREACH(elm, tags2) { DavTag *t = elm->data; if(!ucx_map_cstr_get(map1, t->name)) { equal = 0; break; } i++; } if(i != map1->count) { equal = 0; } ucx_map_free(map1); return equal; } int sync_store_metadata(SyncDirectory *dir, const char *path, LocalResource *local, DavResource *res) { int ret = 0; DavXmlNode *fileinfo = dav_get_property_ns(res, DAV_NS, "finfo"); if(fileinfo) { FileInfo f; finfo_get_values(fileinfo, &f); if(dir->metadata & FINFO_DATE == FINFO_DATE && f.date_set) { // set mtime struct utimbuf t; t.actime = f.last_modified; t.modtime = f.last_modified; if(utime(path, &t)) { fprintf(stderr, "utime failed for file: %s : %s\n", path, strerror(errno)); ret = 1; } } if(dir->metadata & FINFO_MODE == FINFO_MODE && f.mode_set) { // set mode if(chmod(path, f.mode)) { fprintf(stderr, "chmod failed for file: %s : %s\n", path, strerror(errno)); ret = 1; } } } DavXmlNode *xattr_prop = dav_get_property_ns(res, DAV_NS, "xattributes"); if(xattr_prop) { XAttributes *xattr = xml_get_attributes(xattr_prop); if(xattr) { if(!sync_store_xattr(dir, path, xattr)) { if(local->xattr_hash) { free(local->xattr_hash); } local->xattr_hash = xattr->hash; } } } if(sync_store_tags(dir, path, local, res)) { ret = 1; } return ret; } int sync_store_xattr(SyncDirectory *dir, const char *path, XAttributes *xattr) { for(int i=0;i<xattr->nattr;i++) { sstr_t value = xattr->values[i]; if(xattr_set(path, xattr->names[i], value.ptr, value.length)) { fprintf( stderr, "Cannot store xattr '%s' for file: %s\n", xattr->names[i], path); } } return 0; } int sync_store_tags(SyncDirectory *dir, const char *path, LocalResource *local, DavResource *res) { if(!dir->tagconfig) { return 0; } UcxList *tags = NULL; if(dir->tagconfig) { DavXmlNode *tagsprop = dav_get_property_ns(res, DAV_NS, "tags"); if(tagsprop) { tags = parse_dav_xml_taglist(tagsprop); } } DavBool store_tags = FALSE; DavBool tags_changed = FALSE; UcxList *local_tags = sync_get_file_tags(dir, local, &tags_changed, NULL); if(tags_changed) { switch(dir->tagconfig->conflict) { case TAG_NO_CONFLICT: { store_tags = TRUE; break; } case TAG_KEEP_LOCAL: { store_tags = FALSE; break; } case TAG_KEEP_REMOTE: { store_tags = TRUE; local->tags_updated = FALSE; break; } case TAG_MERGE: { UcxList *new_tags = merge_tags(local_tags, tags); // TODO: free tags and local_tags tags = new_tags; store_tags = TRUE; // make sure the merged tags will be pushed the next time local->tags_updated = TRUE; break; } } } else { if(!sync_tags_equal(tags, local_tags)) { store_tags = TRUE; } // TODO: free local_tags } if(!store_tags) { return 0; } int ret = sync_store_tags_local(dir, local, path, tags); // TODO: free stuff return ret; } int sync_store_tags_local(SyncDirectory *dir, LocalResource *local, const char *path, UcxList *tags) { int ret = 0; if(dir->tagconfig->store == TAG_STORE_XATTR) { UcxBuffer *data = NULL; if(tags) { switch(dir->tagconfig->local_format) { default: break; case TAG_FORMAT_TEXT: { data = create_text_taglist(tags); break; } case TAG_FORMAT_CSV: { data = create_csv_taglist(tags); break; } case TAG_FORMAT_MACOS: { data = create_macos_taglist(tags); break; } } if(data) { char *data_hash = dav_create_hash(data->space, data->size); int update = 1; if(local) { if(!local->tags_hash || strcmp(data_hash, local->tags_hash)) { //printf("update: %s\n", local->path); } else { update = 0; } } if(update) { ret = xattr_set(path, dir->tagconfig->xattr_name, data->space, data->pos); if(local) { if(local->tags_hash) { free(local->tags_hash); } local->tags_hash = data_hash; } } else { free(data_hash); } ucx_buffer_free(data); } else { ret = -1; } } else { if(local) { //printf("update: %s\n", local->path); } // ignore errors on remove xattr_remove(path, dir->tagconfig->xattr_name); } } if(!ret) { local->tags_updated = 0; } return ret; } UcxBuffer* sync_get_file_tag_data(SyncDirectory *dir, LocalResource *res) { if(!dir->tagconfig) { return NULL; } if(res->cached_tags) { return res->cached_tags; } UcxBuffer *buf = NULL; if(dir->tagconfig->store == TAG_STORE_XATTR) { ssize_t tag_length = 0; char *local_path = util_concat_path(dir->path, res->path); char* tag_data = xattr_get( local_path, dir->tagconfig->xattr_name, &tag_length); free(local_path); if(tag_length > 0) { buf = ucx_buffer_new(tag_data, (size_t)tag_length, UCX_BUFFER_AUTOFREE); buf->size = (size_t)tag_length; } } res->cached_tags = buf; return buf; } UcxList* sync_get_file_tags(SyncDirectory *dir, LocalResource *res, DavBool *changed, char **newhash) { if(changed) *changed = FALSE; UcxList *tags = NULL; if(!res) { return NULL; } if(!dir->tagconfig) { return NULL; } if(changed && res->tags_updated) { *changed = TRUE; } if(dir->tagconfig->store == TAG_STORE_XATTR) { UcxBuffer *tag_buf = res->cached_tags ? res->cached_tags : sync_get_file_tag_data(dir, res); if(tag_buf) { char *new_hash = dav_create_hash(tag_buf->space, tag_buf->size); if(res->tags_hash) { if(changed && strcmp(res->tags_hash, new_hash)) { *changed = TRUE; } free(res->tags_hash); } else { if(changed) *changed = TRUE; } if(!newhash) { *newhash = new_hash; } else { free(newhash); } switch(dir->tagconfig->local_format) { default: break; case TAG_FORMAT_TEXT: { tags = parse_text_taglist(tag_buf->space, tag_buf->size); break; } case TAG_FORMAT_CSV: { tags = parse_csv_taglist(tag_buf->space, tag_buf->size); break; } case TAG_FORMAT_MACOS: { tags = parse_macos_taglist(tag_buf->space, tag_buf->size); break; } } res->cached_tags = tag_buf; } else if(res->tags_hash) { if(changed) *changed = TRUE; } } return tags; } static int file_seek(FILE *f, curl_off_t offset, int origin) { int ret = fseek(f, offset, origin); return ret == 0 ? CURL_SEEKFUNC_OK : CURL_SEEKFUNC_CANTSEEK; } size_t myread(void *ptr, size_t size, size_t nmemb, FILE *f) { size_t ret = fread(ptr, size, nmemb, f); return ret; } int gen_random_name(char *buf, size_t len) { unsigned char name_prefix[8]; memset(name_prefix, 0, 8); dav_rand_bytes(name_prefix, 8); char *pre = util_hexstr(name_prefix, 8); int64_t ts = (int64_t)time(NULL); int w = snprintf(buf, len, "%"PRId64"-%s", ts, pre); free(pre); return w >= len; } #define VBEGIN_ERROR_MKCOL 1 #define VBEGIN_ERROR_MOVE 2 #define VBEGIN_ERROR_PROPPATCH 3 #define VBEGIN_ERROR_CHECKOUT 4 int versioning_begin(SyncDirectory *dir, DavResource *res) { int ret = 0; if(dir->versioning->type == VERSIONING_SIMPLE && res->exists) { DavResource *history_collection = dav_resource_new( res->session, dir->versioning->collection); // get the path to the version history collection for this resource // if propfind fails we just assume that it doesn't exist // better error handling is done later (sync_put_resource) // if there is no history collection for this resource, we create one char *history_href = NULL; char *vcol_path = dav_get_string_property_ns(res, DAV_NS, VERSION_PATH_PROPERTY); if(!vcol_path) { DavResource *history_res = NULL; // create a new collection for version history // the name is a combination of a random prefix and a timestamp while(!history_res) { char history_res_name[128]; gen_random_name(history_res_name, 128); history_res = dav_resource_new_child( res->session, history_collection, history_res_name); if(dav_exists(history_res)) { dav_resource_free(history_res); history_res = NULL; } } history_res->iscollection = TRUE; if(dav_create(history_res)) { dav_resource_free(history_res); dav_resource_free(history_collection); return VBEGIN_ERROR_MKCOL; } history_href = strdup(history_res->href); dav_resource_free(history_res); } else { history_href = vcol_path; } // find a free url and move 'res' to this location DavResource *version_res = NULL; while(!version_res) { char version_name[128]; gen_random_name(version_name, 128); char *href = util_concat_path(history_href, version_name); version_res = dav_resource_new_href(res->session, href); free(href); char *dest = util_get_url(res->session, version_res->href); int err = dav_moveto(res, dest, FALSE); free(dest); if(err) { dav_resource_free(version_res); version_res = NULL; if(res->session->error != DAV_PRECONDITION_FAILED) { ret = VBEGIN_ERROR_MOVE; break; } } } if(!ret) { dav_set_string_property_ns(version_res, DAV_NS, "origin", res->href); if(dav_store(version_res)) { ret = VBEGIN_ERROR_PROPPATCH; } dav_resource_free(version_res); // we can just set the property here and don't need dav_store // because sync_put_resource will call dav_store(res) later dav_set_string_property_ns( res, DAV_NS, VERSION_PATH_PROPERTY, history_href); } if(vcol_path != history_href) { free(history_href); } dav_resource_free(history_collection); } else if(dir->versioning->type == VERSIONING_DELTAV){ // DeltaV is so much easier :) if(dav_checkout(res)) { ret = VBEGIN_ERROR_CHECKOUT; } } return ret; } int versioning_end(SyncDirectory *dir, DavResource *res) { if(dir->versioning->type == VERSIONING_DELTAV) { return dav_checkin(res); } else { return 0; } } int versioning_delete(SyncDirectory *dir, DavResource *res) { if(dir->versioning->type == VERSIONING_SIMPLE) { // TODO } return 0; } static void update_metadata_hashes(LocalResource *local, MetadataHashes hashes) { if(hashes.tags) { if(local->tags_hash) { free(local->tags_hash); } local->tags_hash = hashes.tags; } if(hashes.tags_remote) { if(local->remote_tags_hash) { free(local->remote_tags_hash); } local->remote_tags_hash = hashes.tags_remote; } if(hashes.xattr) { if(local->xattr_hash) { free(local->xattr_hash); } local->xattr_hash = hashes.xattr; } } int sync_put_resource( SyncDirectory *dir, DavResource *res, LocalResource *local, int *counter) { char *local_path = util_concat_path(dir->path, res->path); SYS_STAT s; if(sys_stat(local_path, &s)) { fprintf(stderr, "Cannot stat file: %s\n", local_path); perror(""); free(local_path); return -1; } FILE *in = sys_fopen(local_path, "rb"); if(!in) { fprintf(stderr, "Cannot open file %s\n", local_path); free(local_path); return -1; } dav_set_content(res, in, (dav_read_func)myread, (dav_seek_func)file_seek); dav_set_content_length(res, s.st_size); MetadataHashes hashes; hashes = sync_set_metadata_properties(dir, res->session, res, local); // before sync_put_resource, remote_resource_is_changed does a propfind // and sets res->exists int exists = res->exists; if(dir->versioning && dir->versioning->always) { int err = versioning_begin(dir, res); if(err) { fprintf(stderr, "Cannot store version for resource: %s\n", res->href); free(local_path); return -1; } } int ret = -1; for(int i=0;i<=dir->max_retry;i++) { if(!exists && dav_create(res)) { continue; } exists = 1; if(dav_store(res)) { continue; } ret = 0; break; } if(dir->versioning && dir->versioning->always) { if(versioning_end(dir, res)) { fprintf(stderr, "Cannot checkin resource\n"); ret = 1; } } if(ret == 0) { (*counter)++; update_metadata_hashes(local, hashes); // check contentlength and get new etag DavResource *up_res = dav_get(res->session, res->path, "D:getetag,idav:status,idav:tags"); if(up_res) { // the new content length must be equal or greater than the file size if(up_res->contentlength < s.st_size) { fprintf(stderr, "Incomplete Upload: %s\n", local_path); ret = -1; // try to set the resource status to 'broken' sync_set_status(res, "broken"); } else { // everything seems fine, we can update the local resource char *etag = dav_get_string_property(up_res, "D:getetag"); if(etag) { if(strlen(etag) > 2 && etag[0] == 'W' && etag[1] == '/') { etag = etag + 2; } } if(local->etag) { free(local->etag); } if(etag) { local->etag = strdup(etag); } else { local->etag = NULL; } if(dav_get_string_property(up_res, "idav:status")) { sync_remove_status(up_res); } dav_resource_free(up_res); } } } else { ret = -1; sync_set_status(res, "broken"); } fclose(in); free(local_path); return ret; } int sync_mkdir(SyncDirectory *dir, DavResource *res, LocalResource *local) { res->iscollection = 1; int ret = -1; for(int i=0;i<=dir->max_retry;i++) { if(dav_create(res)) { continue; } ret = 0; break; } return ret; } int sync_delete_remote_resource( SyncDirectory *dir, DavSession *sn, LocalResource *local_res, int *counter, UcxList **cols) { DavResource *res = dav_get(sn, local_res->path, "D:getetag"); if(!res) { return sn->error == DAV_NOT_FOUND ? 0 : 1; } int ret = 0; sn->error = DAV_OK; if(res->iscollection) { if(cols) { *cols = ucx_list_append(*cols, local_res); } else if(!res->children) { printf("delete: %s\n", res->path); if(dav_delete(res)) { ret = 1; fprintf(stderr, "Cannot delete collection %s\n", res->path); } else { (*counter)++; } } } else { char *etag = dav_get_string_property(res, "D:getetag"); if(etag) { if(strlen(etag) > 2 && etag[0] == 'W' && etag[1] == '/') { etag = etag + 2; } } if(etag && !strcmp(etag, local_res->etag)) { // local resource metadata == remote resource metadata // resource can be deleted printf("delete: %s\n", res->path); if(dir->versioning && dir->versioning->always) { if(versioning_delete(dir, res)) { fprintf( stderr, "Cannot save resource version before deletion\n"); ret = 1; } } if(!ret && dav_delete(res)) { if(sn->error != DAV_NOT_FOUND) { fprintf(stderr, "Cannot delete resource %s\n", res->path); ret = 1; } } else { (*counter)++; } } } // cleanup dav_resource_free(res); return ret; } MetadataHashes sync_set_metadata_properties( SyncDirectory *dir, DavSession *sn, DavResource *res, LocalResource *local) { MetadataHashes hashes = {NULL, NULL, NULL}; if(dir->tagconfig) { // get local tags DavBool changed = 0; char *tags_hash = NULL; UcxList *tags = sync_get_file_tags(dir, local, &changed, &tags_hash); if(changed || local->tags_updated) { hashes.tags = tags_hash; DavBool store_tags = TRUE; // get remote tags UcxList *remote_tags = NULL; DavXmlNode *tagsprop = dav_get_property_ns(res, DAV_NS, "tags"); if(tagsprop) { remote_tags = parse_dav_xml_taglist(tagsprop); } char *remote_hash = create_tags_hash(remote_tags); if(nullstrcmp(remote_hash, local->remote_tags_hash)) { // the tags have changed on the server switch(dir->tagconfig->conflict) { case TAG_NO_CONFLICT: break; case TAG_KEEP_LOCAL: break; case TAG_KEEP_REMOTE: { store_tags = FALSE; local->tags_updated = FALSE; break; } case TAG_MERGE: { UcxList *new_tags = merge_tags(tags, remote_tags); free_taglist(tags); tags = new_tags; break; } } } if(dir->tagconfig->local_format == TAG_FORMAT_CSV) { // csv tag lists don't have colors, so we have to add // the colors from the remote tag list add_tag_colors(tags, remote_tags); } if(store_tags) { if(tags) { DavXmlNode *tagprop = create_xml_taglist(tags); dav_set_property_ns(res, DAV_NS, "tags", tagprop); } else { dav_remove_property_ns(res, DAV_NS, "tags"); } } free_taglist(remote_tags); } else { if(tags_hash) { free(tags_hash); } } free_taglist(tags); } if(local->finfo_updated) { struct stat s; s.st_mode = local->mode; s.st_mtime = local->last_modified; s.st_uid = local->uid; s.st_gid = local->gid; resource_set_finfo_s(&s, res, dir->metadata); } if(local->xattr_updated) { if(local->xattr) { resource_set_xattr(res, local->xattr); hashes.xattr = strdup(local->xattr->hash); } else { dav_remove_property(res, "idav:xattributes"); } } return hashes; } int sync_update_metadata( SyncDirectory *dir, DavSession *sn, DavResource *res, LocalResource *local) { MetadataHashes hashes = sync_set_metadata_properties(dir, sn, res, local); int err = 0; printf("update: %s\n", local->path); if(dav_store(res)) { print_resource_error(sn, local->path); err = 1; } else { update_metadata_hashes(local, hashes); } return err; } void remove_deleted_conflicts(SyncDirectory *dir, SyncDatabase *db) { char **dc = calloc(sizeof(void*), db->conflict->count); int numdc = 0; UcxMapIterator i = ucx_map_iterator(db->conflict); LocalResource *res; UCX_MAP_FOREACH(key, res, i) { char *path = util_concat_path(dir->path, res->path); SYS_STAT s; if(sys_stat(path, &s)) { if(errno == ENOENT) { dc[numdc] = res->path; numdc++; } else { fprintf(stderr, "Cannot stat file: %s\n", path); perror(""); } } free(path); } for(int i=0;i<numdc;i++) { ucx_map_cstr_remove(db->conflict, dc[i]); } free(dc); } static void resolve_skipped(SyncDatabase *db) { UcxKey k; LocalResource *res; UcxMapIterator i = ucx_map_iterator(db->resources); int skipped = 0; UCX_MAP_FOREACH(k, res, i) { if(res->skipped) { skipped++; fprintf(stderr, "skipped from push: %s\n", res->path); } } if(skipped > 0) { fprintf(stderr, " To resolve conflict resources skipped by push run dav-sync pull first\n" " before resolve-conflicts or delete-conflicts.\n\n"); } } int cmd_resolve_conflicts(CmdArgs *a) { if(a->argc != 1) { fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many"); return -1; } SyncDirectory *dir = scfg_get_dir(a->argv[0]); if(!dir) { fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]); return -1; } if(scfg_check_dir(dir)) { return -1; } SyncDatabase *db = load_db(dir->database); if(!db) { fprintf(stderr, "Cannot load database file: %s\n", dir->database); return -1; } resolve_skipped(db); int ret = 0; // remove conflicts int num_conflict = db->conflict->count; ucx_map_free_content(db->conflict, (ucx_destructor)local_resource_free); ucx_map_clear(db->conflict); // store db if(store_db(db, dir->database, dir->db_settings)) { fprintf(stderr, "Cannot store sync db\n"); fprintf(stderr, "Abort\n"); ret = -2; } // cleanup destroy_db(db); // Report if(ret != -2) { char *str_conflict = num_conflict == 1 ? "conflict" : "conflicts"; printf("Result: %d %s resolved\n", num_conflict, str_conflict); } return ret; } int cmd_delete_conflicts(CmdArgs *a) { if(a->argc != 1) { fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many"); return -1; } SyncDirectory *dir = scfg_get_dir(a->argv[0]); if(!dir) { fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]); return -1; } if(scfg_check_dir(dir)) { return -1; } SyncDatabase *db = load_db(dir->database); if(!db) { fprintf(stderr, "Cannot load database file: %s\n", dir->database); return -1; } resolve_skipped(db); int num_del = 0; int num_err = 0; int ret = 0; // delete all conflict files UcxMapIterator i = ucx_map_iterator(db->conflict); LocalResource *res; UCX_MAP_FOREACH(key, res, i) { printf("delete: %s\n", res->path); char *path = util_concat_path(dir->path, res->path); if(sys_unlink(path)) { if(errno != ENOENT) { perror("unlink"); num_err++; } } else { num_del++; } free(path); } ucx_map_free_content(db->conflict, (ucx_destructor)local_resource_free); ucx_map_clear(db->conflict); // store db if(store_db(db, dir->database, dir->db_settings)) { fprintf(stderr, "Cannot store sync db\n"); fprintf(stderr, "Abort\n"); ret = -1; } // cleanup destroy_db(db); // Report if(ret == 0) { char *str_delete = num_del == 1 ? "file" : "files"; char *str_error = num_err == 1 ? "error" : "errors"; printf("Result: %d conflict %s deleted, %d %s\n", num_del, str_delete, num_err, str_error); } return ret; } // TODO: remove code dup (main.c ls_size_str) static char* size_str(uint64_t size) { char *str = malloc(16); if(size < 0x400) { snprintf(str, 16, "%" PRIu64 " bytes", size); } else if(size < 0x100000) { float s = (float)size/0x400; int diff = (s*100 - (int)s*100); if(diff > 90) { diff = 0; s += 0.10f; } if(size < 0x2800 && diff != 0) { // size < 10 KiB snprintf(str, 16, "%.1f KiB", s); } else { snprintf(str, 16, "%.0f KiB", s); } } else if(size < 0x40000000) { float s = (float)size/0x100000; int diff = (s*100 - (int)s*100); if(diff > 90) { diff = 0; s += 0.10f; } if(size < 0xa00000 && diff != 0) { // size < 10 MiB snprintf(str, 16, "%.1f MiB", s); } else { size /= 0x100000; snprintf(str, 16, "%.0f MiB", s); } } else if(size < 0x1000000000ULL) { float s = (float)size/0x40000000; int diff = (s*100 - (int)s*100); if(diff > 90) { diff = 0; s += 0.10f; } if(size < 0x280000000 && diff != 0) { // size < 10 GiB snprintf(str, 16, "%.1f GiB", s); } else { size /= 0x40000000; snprintf(str, 16, "%.0f GiB", s); } } else { size /= 1024; float s = (float)size/0x40000000; int diff = (s*100 - (int)s*100); if(diff > 90) { diff = 0; s += 0.10f; } if(size < 0x280000000 && diff != 0) { // size < 10 TiB snprintf(str, 16, "%.1f TiB", s); } else { size /= 0x40000000; snprintf(str, 16, "%.0f TiB", s); } } return str; } int cmd_list_versions(CmdArgs *a) { if(a->argc != 1) { fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many"); return -1; } SyncFile file; int ret = 0; char *path = a->argv[0]; int err = sync_get_file(a, path, &file, TRUE); if(err) { sync_print_get_file_err(path, err); return 1; } SyncDirectory *dir = file.dir; if(!dir->versioning) { fprintf(stderr, "No versioning configured for syncdir %s\n", dir->name); } Repository *repo = get_repository(sstr(dir->repository)); if(!repo) { fprintf(stderr, "Unknown repository %s\n", dir->repository); return -1; } SyncDatabase *db = load_db(dir->database); if(!db) { fprintf(stderr, "Cannot load database file: %s\n", dir->database); return -1; } remove_deleted_conflicts(dir, db); DavSession *sn = create_session(ctx, repo, dir->collection); ucx_mempool_reg_destr(sn->mp, db, (ucx_destructor)destroy_db); if (cmd_getoption(a, "verbose")) { curl_easy_setopt(sn->handle, CURLOPT_VERBOSE, 1L); curl_easy_setopt(sn->handle, CURLOPT_STDERR, stderr); } DavResource *res = dav_resource_new(sn, file.path); if(dir->versioning->type == VERSIONING_SIMPLE) { do { DavPropName p; p.ns = DAV_NS; p.name = VERSION_PATH_PROPERTY; if(dav_load_prop(res, &p, 1)) { print_resource_error(sn, file.path); ret = 1; break; } char *vcol_href = dav_get_string_property_ns(res, DAV_NS, VERSION_PATH_PROPERTY); if(!vcol_href) { ret = 1; break; } DavResource *vcol = dav_resource_new_href(sn, vcol_href); if(!vcol) { ret = 1; break; } if(dav_load_prop(vcol, NULL, 0)) { print_resource_error(sn, vcol->path); ret = 1; break; } DavResource *child = vcol->children; UcxList *children = NULL; while(child) { children = ucx_list_append(children, child); child = child->next; } children = ucx_list_sort(children, ucx_cmp_str, NULL); DavBool first = 1; UCX_FOREACH(elm, children) { DavResource *c = elm->data; time_t now = c->lastmodified; struct tm *date = gmtime(&now); char str[32]; putenv("LC_TIME=C"); size_t len = strftime(str, 32, "%a, %d %b %Y %H:%M:%S GMT", date); if(!first) { putchar('\n'); } printf("name: %s\n", c->name); printf("lastmodified: %s\n", str); char *server = util_url_base(sn->base_url); char *url = util_concat_path(server, c->href); printf("url: %s\n", url); free(server); free(url); first = 0; } ucx_list_free(children); } while(0); } free(file.path); dav_session_destroy(sn); return ret; } int cmd_trash_info(CmdArgs *a) { if(a->argc != 1) { fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many"); return -1; } SyncDirectory *syncdir = scfg_get_dir(a->argv[0]); if(!syncdir) { fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]); return -1; } if(scfg_check_dir(syncdir)) { return -1; } if(!syncdir->trash) { printf("trash not configured for %s\n", syncdir->name); return 0; } SYS_DIR dir = sys_opendir(syncdir->trash); if(!dir) { fprintf(stderr, "cannot open trash directory: %s\n", syncdir->trash); perror("opendir"); return -1; } uint64_t trashsize = 0; int count = 0; SysDirEnt *ent; while((ent = sys_readdir(dir)) != NULL) { if(!strcmp(ent->name, ".") || !strcmp(ent->name, "..")) { continue; } char *path = util_concat_path(syncdir->trash, ent->name); SYS_STAT s; if(sys_stat(path, &s)) { perror("stat"); } else { trashsize += s.st_size; } count++; free(path); } sys_closedir(dir); printf("path: %s\n", syncdir->trash); printf("%d %s\n", count, count == 1 ? "file" : "files"); char *sizestr = size_str(trashsize); printf("%s\n", sizestr); free(sizestr); return 0; } int cmd_empty_trash(CmdArgs *a) { if(a->argc != 1) { fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many"); return -1; } SyncDirectory *syncdir = scfg_get_dir(a->argv[0]); if(!syncdir) { fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]); return -1; } if(!syncdir->trash) { fprintf(stderr, "trash not configured for %s\n", syncdir->name); return -1; } SYS_DIR dir = sys_opendir(syncdir->trash); if(!dir) { fprintf(stderr, "cannot open trash directory: %s\n", syncdir->trash); perror("opendir"); return -1; } SysDirEnt *ent; while((ent = sys_readdir(dir)) != NULL) { if(!strcmp(ent->name, ".") || !strcmp(ent->name, "..")) { continue; } char *path = util_concat_path(syncdir->trash, ent->name); printf("delete: %s\n", path); SYS_STAT s; if(sys_stat(path, &s)) { perror("stat"); free(path); continue; } if(S_ISDIR(s.st_mode)) { if(rmdir(path)) { perror("rmdir"); } } else { if(sys_unlink(path)) { perror("unlink"); } } free(path); } sys_closedir(dir); return 0; } #define CMD_TAG_ADD 0 #define CMD_TAG_REMOVE 1 #define CMD_TAG_SET 2 #define CMD_TAG_LIST 3 int cmd_add_tag(CmdArgs *args) { if(args->argc != 2) { fprintf(stderr, "Too %s arguments\n", args->argc <= 1 ? "few" : "many"); return -1; } return cmd_tagop(args, CMD_TAG_ADD); } int cmd_remove_tag(CmdArgs *args) { if(args->argc != 2) { fprintf(stderr, "Too %s arguments\n", args->argc <= 1 ? "few" : "many"); return -1; } return cmd_tagop(args, CMD_TAG_REMOVE); } int cmd_set_tags(CmdArgs *args) { if(args->argc < 1 || args->argc > 2) { fprintf(stderr, "Too %s arguments\n", args->argc < 1 ? "few" : "many"); return -1; } return cmd_tagop(args, CMD_TAG_SET); } int cmd_list_tags(CmdArgs *args) { if(args->argc != 1) { fprintf(stderr, "Too %s arguments\n", args->argc <= 1 ? "few" : "many"); return -1; } return cmd_tagop(args, CMD_TAG_LIST); } int cmd_tagop(CmdArgs *args, int cmd) { SyncFile file; int ret = 0; char *path = args->argv[0]; int err = sync_get_file(args, path, &file, TRUE); if(err) { sync_print_get_file_err(path, err); return -1; } if(!file.dir->tagconfig) { fprintf(stderr, "Tags are not supported for this sync directory\n"); return -1; } SyncDatabase *db = load_db(file.dir->database); if(!db) { fprintf(stderr, "Cannot load sync directory database\n"); return -1; } LocalResource *localres = ucx_map_cstr_get(db->resources, file.path); UcxList *tags = NULL; DavBool store_tags = FALSE; if(cmd != CMD_TAG_SET) { char *tag = args->argv[1]; char *tagcolor = NULL; // TODO: get color tags = sync_get_file_tags(file.dir, localres, NULL, NULL); UcxList *x = NULL; UCX_FOREACH(elm, tags) { DavTag *t = elm->data; if(cmd == CMD_TAG_LIST) { printf("%s\n", t->name); } else if(!strcmp(t->name, tag)) { x = elm; break; } } if(cmd == CMD_TAG_ADD) { if(!x) { DavTag *newtag = malloc(sizeof(DavTag)); newtag->name = tag; newtag->color = tagcolor; tags = ucx_list_append(tags, newtag); store_tags = TRUE; } } else if(cmd == CMD_TAG_REMOVE) { if(tags) { tags = ucx_list_remove(tags, x); } store_tags = TRUE; } } else { if(args->argc == 2) { char *tags_str = args->argv[1]; tags = parse_csv_taglist(tags_str, strlen(tags_str)); store_tags = TRUE; // TODO: read from stdin if tags_str is "-" } else if (args->argc == 1) { store_tags = TRUE; } else { fprintf(stderr, "Too many arguments\n"); ret = -1; } } if(store_tags) { if(sync_store_tags_local(file.dir, NULL, path, tags)) { fprintf(stderr, "Cannot store tags\n"); } if(localres) { localres->tags_updated = TRUE; if(!tags) { if(localres->tags_hash) { free(localres->tags_hash); } localres->tags_hash = NULL; } } } // store db if(store_db(db, file.dir->database, file.dir->db_settings)) { fprintf(stderr, "Cannot store sync db\n"); ret = -2; } free(file.path); return ret; } int isfileindir(SyncDirectory *dir, const char *path, SyncFile *f) { char *fullpath; if(path[0] != '/') { size_t wdlen = 256; char *wd = malloc(wdlen); while(!getcwd(wd, wdlen)) { if(errno == ERANGE) { wdlen *= 2; char *newbuf = realloc(wd, wdlen); if (newbuf) { wd = newbuf; } else { free(wd); return 0; } } else { free(wd); return 0; } } fullpath = util_concat_path(wd, path); free(wd); } else { fullpath = strdup(path); } // TODO: normalize path if(!sstrprefix(sstr((char*)fullpath), sstr(dir->path))) { free(fullpath); return 0; } // TODO: check filter f->dir = dir; f->path = util_concat_path("/", fullpath + strlen(dir->path)); free(fullpath); return 1; } int sync_get_file(CmdArgs *args, const char *path, SyncFile *f, DavBool dostat) { if(dostat) { SYS_STAT s; if(sys_stat(path, &s)) { switch(errno) { case EACCES: return 2; case ENOENT: return 1; default: return 3; } } } char *sdir = cmd_getoption(args, "syncdir"); if(sdir) { SyncDirectory *dir = scfg_get_dir(sdir); if(!dir) { return 6; } if(!isfileindir(dir, path, f)) { return 4; } } else { SyncDirectory *target = NULL; UcxMapIterator i = scfg_directory_iterator(); UcxKey k; SyncDirectory *dir; UCX_MAP_FOREACH(key, dir, i) { if(isfileindir(dir, path, f)) { if(target) { return 5; } else { target = dir; } } } if(!target) { return 4; } } return 0; } void sync_print_get_file_err(const char *path, int err) { switch(err) { case 1: fprintf(stderr, "File %s: not found\n", path); break; case 2: fprintf(stderr, "File %s: permission denied\n"); break; case 3: fprintf(stderr, "File %s: stat failed: %s\n", path, strerror(errno)); break; case 4: fprintf(stderr, "File %s is not in any syncdir\n"); break; case 5: fprintf(stderr, "File %s is in multiple syncdirs\n"); break; case 6: fprintf(stderr, "Syncdir not found\n"); break; } } int cmd_add_directory(CmdArgs *args) { if(!get_repositories()) { fprintf(stderr, "No repositories available. Run 'dav add-repository' first.\n"); fprintf(stderr, "Abort\n"); return -1; } printf("Each sync directory must have an unique name.\n"); char *name = assistant_getcfg("name"); if(!name) { fprintf(stderr, "Abort\n"); return -1; } if(scfg_get_dir(name)) { fprintf(stderr, "Directory %s already exists.\nAbort\n", name); return -1; } printf("Enter local directory path.\n"); char *path = assistant_getcfg("path"); if(!path) { fprintf(stderr, "Abort\n"); return -1; } printf("Specify webdav repository.\n"); UcxList *repos = get_repositories(); int i = 0; UCX_FOREACH(elm, repos) { Repository *r = elm->data; printf("%d) %s\n", i, r->name); i++; } char *repository = assistant_getcfg("repository"); char *reponame = NULL; if(!repository) { fprintf(stderr, "Abort\n"); return -1; } int64_t reponum = 0; if(util_strtoint(repository, &reponum)) { if(reponum < 0) { fprintf(stderr, "Wrong input.\nAbort\n"); return -1; } UcxList *elm = ucx_list_get(repos, reponum); if(elm) { Repository *r = elm->data; reponame = r->name; } else { fprintf(stderr, "Wrong input.\nAbort\n"); return -1; } } else { if(get_repository(sstr(repository))) { reponame = repository; } else { fprintf(stderr, "Repository %s doesn't exist.\nAbort\n", repository); return -1; } } printf("Enter collection relative to the repository base url.\n"); char *collection = assistant_getdefcfg("collection", "/"); char *db = generate_db_name(name); SyncDirectory dir; memset(&dir, 0, sizeof(SyncDirectory)); dir.name = name; dir.path = path; dir.repository = reponame; dir.collection = collection; dir.trash = ".trash"; dir.database = db; int ret = 0; if(add_directory(&dir)) { fprintf(stderr, "Cannot write sync.xml\n"); ret = -1; } else { printf("\nAdded directory: %s (%s)\n", name, path); } free(name); free(path); free(repository); free(collection); free(db); return ret; } int cmd_list_dirs() { UcxMapIterator iter = scfg_directory_iterator(); SyncDirectory *dir; UCX_MAP_FOREACH(key, dir, iter) { printf("%s\n", dir->name); } return 0; } int cmd_check_repositories() { int ret = EXIT_SUCCESS; UcxList *reponames = NULL; { UcxMapIterator iter = scfg_directory_iterator(); SyncDirectory *dir; UCX_MAP_FOREACH(key, dir, iter) { reponames = ucx_list_append(reponames, dir->repository); } } UCX_FOREACH(listelem, reponames) { char *reponame = listelem->data; printf("Checking %s... ", reponame); Repository* repo = get_repository(sstr(reponame)); if (!repo) { printf(" not found in config.xml!\n"); ret = EXIT_FAILURE; } else { DavSession *sn = create_session(ctx, repo, repo->url); if (sn) { DavResource *res = dav_query(sn, "select - from / with depth = 0"); if (res) { printf("OK.\n"); dav_resource_free(res); } else { printf("unavailable!\n"); ret = EXIT_FAILURE; } dav_session_destroy(sn); } else { printf("cannot create session!\n"); ret = EXIT_FAILURE; } } } ucx_list_free(reponames); return ret; } char* create_locktoken_file(const char *syncdirname, const char *locktoken) { sstr_t fname = ucx_sprintf("locktoken-%s.txt", syncdirname); char *path = config_file_path(fname.ptr); free(fname.ptr); FILE *file = sys_fopen(path, "w"); if(file) { fprintf(file, "%s\n", locktoken); fclose(file); return path; } else { perror("Cannot create locktoken file"); free(path); return NULL; } }