Sat, 12 Oct 2019 08:48:35 +0200
remove command aliases from dav help text
/* * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. * * Copyright 2019 Olaf Wintermann. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include <stdio.h> #include <stdlib.h> #include <string.h> #include <errno.h> #include <unistd.h> #include <signal.h> #include <time.h> #include <utime.h> #include <libxml/xmlerror.h> #include <sys/types.h> #include <ucx/string.h> #include <ucx/utils.h> #include <ucx/properties.h> #include <dirent.h> #include <math.h> #include <libidav/webdav.h> #include <libidav/utils.h> #include <libidav/crypto.h> #include <libidav/session.h> #include "sync.h" #include "config.h" #include "sopt.h" #include "error.h" #include "assistant.h" #include "libxattr.h" #include "tags.h" #include "system.h" #include <pthread.h> #include <ctype.h> static DavContext *ctx; static int sync_shutdown = 0; static void xmlerrorfnc(void * c, const char * msg, ... ) { va_list ap; va_start(ap, msg); vfprintf(stderr, msg, ap); va_end(ap); } static DavPropName defprops[] = { { "DAV:", "getetag" }, { DAV_NS, "status" }, { DAV_NS, "content-hash" }, { DAV_NS, "split" }, { DAV_PROPS_NS, "finfo" }, { DAV_PROPS_NS, "tags" }, { DAV_PROPS_NS, "xattributes" }, { DAV_PROPS_NS, "link" } }; static size_t numdefprops = 8 ; /* * strcmp version that works with NULL pointers */ static int nullstrcmp(const char *s1, const char *s2) { if(!s1 && s2) { return -1; } if(s1 && !s2) { return 1; } if(!s1 && !s2) { return 0; } return strcmp(s1, s2); } static char* nullstrdup(const char *s) { return s ? strdup(s) : NULL; } static void nullfree(void *p) { if(p) { free(p); } } int main(int argc, char **argv) { if(argc < 2) { fprintf(stderr, "Missing command\n"); print_usage(argv[0]); return -1; } char *cmd = argv[1]; CmdArgs *args = cmd_parse_args(argc - 2, argv + 2); if(!args) { print_usage(argv[0]); return -1; } int ret = EXIT_FAILURE; if(!strcasecmp(cmd, "version") || !strcasecmp(cmd, "-version") || !strcasecmp(cmd, "--version")) { fprintf(stderr, "dav-sync %s\n", DAV_VERSION); cmd_args_free(args); return -1; } xmlGenericErrorFunc fnc = xmlerrorfnc; initGenericErrorDefaultFunc(&fnc); sys_init(); ctx = dav_context_new(); int cfgret = load_config(ctx) || load_sync_config(); pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_lock(&mutex); pthread_t tid; if(!strcmp(cmd, "check") || !strcmp(cmd, "check-config")) { if(!cfgret) { fprintf(stdout, "Configuration OK.\n"); ret = EXIT_SUCCESS; } else { /* no output, the warnings are written by load_config */ ret = EXIT_FAILURE; } } else if(!cfgret) { if(!strcmp(cmd, "pull")) { tid = start_sighandler(&mutex); ret = cmd_pull(args); stop_sighandler(&mutex, tid); } else if(!strcmp(cmd, "push")) { tid = start_sighandler(&mutex); ret = cmd_push(args, FALSE); stop_sighandler(&mutex, tid); } else if(!strcmp(cmd, "archive")) { tid = start_sighandler(&mutex); ret = cmd_push(args, TRUE); stop_sighandler(&mutex, tid); } else if(!strcmp(cmd, "restore")) { tid = start_sighandler(&mutex); ret = cmd_restore(args); stop_sighandler(&mutex, tid); } else if(!strcmp(cmd, "list-conflicts")) { ret = cmd_list_conflicts(args); } else if(!strcmp(cmd, "resolve-conflicts")) { ret = cmd_resolve_conflicts(args); } else if(!strcmp(cmd, "delete-conflicts")) { ret = cmd_delete_conflicts(args); } else if(!strcmp(cmd, "list-versions")) { ret = cmd_list_versions(args); } else if(!strcmp(cmd, "trash-info")) { ret = cmd_trash_info(args); } else if(!strcmp(cmd, "empty-trash")) { ret = cmd_empty_trash(args); } else if(!strcmp(cmd, "add-tag")) { ret = cmd_add_tag(args); } else if(!strcmp(cmd, "remove-tag")) { ret = cmd_remove_tag(args); } else if(!strcmp(cmd, "set-tags")) { ret = cmd_set_tags(args); } else if(!strcmp(cmd, "list-tags")) { ret = cmd_list_tags(args); } else if(!strcmp(cmd, "add-dir") || !strcmp(cmd, "add-directory")) { ret = cmd_add_directory(args); } else if(!strcmp(cmd, "list-dirs") || !strcmp(cmd, "list-directories")) { ret = cmd_list_dirs(); } else if(!strcmp(cmd, "check-repos") || !strcmp(cmd, "check-repositories")) { ret = cmd_check_repositories(); } else { print_usage(argv[0]); } } // cleanup cmd_args_free(args); dav_context_destroy(ctx); free_config(); free_sync_config(); curl_global_cleanup(); xmlCleanupParser(); sys_uninit(); return ret; } void print_usage(char *cmd) { fprintf(stderr, "Usage: %s command [options] arguments...\n\n", cmd); fprintf(stderr, "Commands:\n"); fprintf(stderr, " pull [-cldr] [-t <tags>] <directory>\n"); fprintf(stderr, " push [-cldrSRM] [-t <tags>] <directory>\n"); fprintf(stderr, " archive [-cldSRM] [-t <tags>] <directory>\n"); fprintf(stderr, " restore [-ldRM] [-V <version>] [-s <directory>] [file...]\n"); fprintf(stderr, " list-conflicts <directory>\n"); fprintf(stderr, " resolve-conflicts <directory>\n"); fprintf(stderr, " delete-conflicts <directory>\n"); fprintf(stderr, " trash-info <directory>\n"); fprintf(stderr, " empty-trash <directory>\n"); fprintf(stderr, " add-tag [-s <syncdir>] <file> <tag>\n"); fprintf(stderr, " remove-tag [-s <syncdir>] <file> <tag>\n"); fprintf(stderr, " set-tags [-s <syncdir>] <file> [tags]\n"); fprintf(stderr, " list-tags [-s <syncdir>] <file>\n\n"); fprintf(stderr, "Options:\n"); fprintf(stderr, " -c Disable conflict detection\n"); fprintf(stderr, " -l Lock the repository before access\n"); fprintf(stderr, " -d Don't lock the repository\n"); fprintf(stderr, " -t <tags> " "Only sync files which have the specified tags\n"); fprintf(stderr, " -r " "Remove resources not matching the tag filter\n"); fprintf(stderr, " -V <vers> Restore specific version\n"); fprintf(stderr, " -S Save previous file version\n"); fprintf(stderr, " -R Restore removed files\n"); fprintf(stderr, " -M Restore modified files\n"); fprintf(stderr, " -v Verbose output (all commands)\n\n"); fprintf(stderr, "Config commands:\n"); fprintf(stderr, " add-directory\n"); fprintf(stderr, " list-directories\n"); fprintf(stderr, " check-config\n"); fprintf(stderr, " check-repositories\n\n"); } static void handlesig(int sig) { if(sync_shutdown) { exit(-1); } fprintf(stderr, "abort\n"); sync_shutdown = 1; } static void* sighandler(void *data) { signal(SIGTERM, handlesig); signal(SIGINT, handlesig); pthread_mutex_t *mutex = data; pthread_mutex_lock(mutex); // block thread return NULL; } pthread_t start_sighandler(pthread_mutex_t *mutex) { pthread_t tid; if(pthread_create(&tid, NULL, sighandler, mutex)) { perror("pthread_create"); exit(-1); } return tid; } void stop_sighandler(pthread_mutex_t *mutex, pthread_t tid) { pthread_mutex_unlock(mutex); void *data; pthread_join(tid, &data); } static char* create_local_path(SyncDirectory *dir, const char *path) { char *local_path = util_concat_path(dir->path, path); size_t local_path_len = strlen(local_path); if(local_path[local_path_len-1] == '/') { local_path[local_path_len-1] = '\0'; } return local_path; } static int res_matches_filter(Filter *filter, char *res_path) { // include/exclude filter UCX_FOREACH(inc, filter->include) { regex_t* pattern = (regex_t*) inc->data; if (regexec(pattern, res_path, 0, NULL, 0) == 0) { UCX_FOREACH(exc, filter->exclude) { regex_t* pattern = (regex_t*) exc->data; if (regexec(pattern, res_path, 0, NULL, 0) == 0) { return 1; } } return 0; } } return 1; } static int res_matches_dir_filter(SyncDirectory *dir, char *res_path) { // trash filter if (dir->trash) { sstr_t rpath = sstr(util_concat_path(dir->path, res_path)); if (util_path_isrelated(dir->trash, rpath.ptr)) { free(rpath.ptr); return 1; } free(rpath.ptr); } // versioning filter if (dir->versioning) { if(util_path_isrelated(dir->versioning->collection, res_path)) { return 1; } } return res_matches_filter(&dir->filter, res_path); } static int res_matches_tags(DavResource *res, SyncTagFilter *tagfilter) { if(!tagfilter || tagfilter->mode == DAV_SYNC_TAGFILTER_OFF) { return 1; } // NOTE: currently not implementable //int scope = res->iscollection ? // DAV_SYNC_TAGFILTER_SCOPE_COLLECTION // : DAV_SYNC_TAGFILTER_SCOPE_RESOURCE; //if((tagfilter->scope & scope) != scope) { // return 1; //} if(res->iscollection) { return 1; } DavXmlNode *tagsprop = dav_get_property_ns(res, DAV_PROPS_NS, "tags"); UcxList *res_tags = parse_dav_xml_taglist(tagsprop); int ret = matches_tagfilter(res_tags, tagfilter); ucx_list_free_content(res_tags, (ucx_destructor) free_dav_tag); ucx_list_free(res_tags); return ret; } static int localres_matches_tags( SyncDirectory *dir, LocalResource *res, SyncTagFilter *tagfilter) { if(!tagfilter || tagfilter->mode == DAV_SYNC_TAGFILTER_OFF) { return 1; } //int scope = res->isdirectory ? // DAV_SYNC_TAGFILTER_SCOPE_COLLECTION // : DAV_SYNC_TAGFILTER_SCOPE_RESOURCE; //if((tagfilter->scope & scope) != scope) { // return 1; //} if(res->isdirectory) { return 1; } DavBool changed = 0; UcxList *res_tags = sync_get_file_tags(dir, res, &changed, NULL); int ret = matches_tagfilter(res_tags, tagfilter); UCX_FOREACH(elm, res_tags) { DavTag *t = elm->data; free_dav_tag(t); } ucx_list_free(res_tags); return ret; } static DavSession* create_session(DavContext *ctx, Repository *repo, char *collection) { int flags = get_repository_flags(repo); char *url = repo->url; DavBool find_collection = TRUE; if((flags & DAV_SESSION_DECRYPT_NAME) != DAV_SESSION_DECRYPT_NAME) { url = util_concat_path(repo->url, collection); find_collection = FALSE; } if(!collection || (collection[0] == '/' && strlen(collection) == 1)) { // collection is NULL or "/" // we don't need to find any collection because the repo url is // the base url find_collection = FALSE; } DavSession *sn = dav_session_new_auth( ctx, url, repo->user, repo->password); if(url != repo->url) { free(url); } sn->flags = flags; sn->key = dav_context_get_key(ctx, repo->default_key); curl_easy_setopt(sn->handle, CURLOPT_HTTPAUTH, repo->authmethods); curl_easy_setopt(sn->handle, CURLOPT_SSLVERSION, repo->ssl_version); if(repo->cert) { curl_easy_setopt(sn->handle, CURLOPT_CAPATH, repo->cert); } if(!repo->verification) { curl_easy_setopt(sn->handle, CURLOPT_SSL_VERIFYPEER, 0); curl_easy_setopt(sn->handle, CURLOPT_SSL_VERIFYHOST, 0); } if(find_collection) { DavResource *col = dav_resource_new(sn, collection); dav_exists(col); // exec this to get the href // we actually don't care what the result is // if it doesn't exists, an error will occur later // and we can't handle it here char *newurl = util_concat_path(repo->url, util_resource_name(col->href)); dav_session_set_baseurl(sn, newurl); free(newurl); } return sn; } static void print_allowed_cmds(SyncDirectory *dir) { fprintf(stderr, "Allowed commands: "); char *sep = ""; if((dir->allow_cmd & SYNC_CMD_PULL) == SYNC_CMD_PULL) { fprintf(stderr, "pull"); sep = ", "; } if((dir->allow_cmd & SYNC_CMD_PUSH) == SYNC_CMD_PUSH) { fprintf(stderr, "%spush", sep); sep = ", "; } if((dir->allow_cmd & SYNC_CMD_ARCHIVE) == SYNC_CMD_ARCHIVE) { fprintf(stderr, "%sarchive", sep); } fprintf(stderr, "\n"); } static void localres_keep(SyncDatabase *db, const char *path) { LocalResource *local = ucx_map_cstr_remove(db->resources, path); if(local) { local->keep = TRUE; } } static int xattr_filter(const char *name, SyncDirectory *dir) { // exclude tag xattr if( dir->tagconfig && dir->tagconfig->store == TAG_STORE_XATTR && !strcmp(dir->tagconfig->xattr_name, name)) { return 0; } return 1; } void res2map(DavResource *root, UcxMap *map) { UcxList *stack = ucx_list_prepend(NULL, root->children); while(stack) { DavResource *res = stack->data; stack = ucx_list_remove(stack, stack); while(res) { ucx_map_cstr_put(map, res->path, res); if(res->children) { stack = ucx_list_prepend(stack, res->children); } res = res->next; } } } int cmd_pull(CmdArgs *a) { if(a->argc != 1) { fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many"); return -1; } // if there are syntax errors in the command line, fail asap. SyncTagFilter* tagfilter = parse_tagfilter_string( cmd_getoption(a, "tags"), DAV_SYNC_TAGFILTER_SCOPE_RESOURCE); if (!tagfilter) { fprintf(stderr, "Malformed tag filter\n"); return -1; } // TODO: tons of memory leaks... // call free_tagfilter() before each return SyncDirectory *dir = scfg_get_dir(a->argv[0]); if(!dir) { fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]); return -1; } if(scfg_check_dir(dir)) { return -1; } if((dir->allow_cmd & SYNC_CMD_PULL) != SYNC_CMD_PULL) { fprintf(stderr, "Command 'pull' is not allowed for this sync dir\n"); print_allowed_cmds(dir); return -1; } Repository *repo = get_repository(sstr(dir->repository)); if(!repo) { fprintf(stderr, "Unknown repository %s\n", dir->repository); return -1; } SyncDatabase *db = load_db(dir->database); if(!db) { fprintf(stderr, "Cannot load database file: %s\n", dir->database); return -1; } remove_deleted_conflicts(dir, db); UcxMap *hashes = NULL; if(SYNC_HASHING(dir)) { hashes = create_hash_index(db); } DavSession *sn = create_session(ctx, repo, dir->collection); ucx_mempool_reg_destr(sn->mp, db, (ucx_destructor)destroy_db); if (cmd_getoption(a, "verbose")) { curl_easy_setopt(sn->handle, CURLOPT_VERBOSE, 1L); curl_easy_setopt(sn->handle, CURLOPT_STDERR, stderr); } // lock repository char *locktokenfile = NULL; DavBool locked = FALSE; DavResource *root = dav_resource_new(sn, "/"); root->iscollection = TRUE; if((dir->lockpush || cmd_getoption(a, "lock")) && !cmd_getoption(a, "nolock")) { if(dav_lock_t(root, dir->lock_timeout)) { print_resource_error(sn, "/"); dav_session_destroy(sn); fprintf(stderr, "Abort\n"); return -1; } DavLock *lock = dav_get_lock(sn, "/"); if(lock) { printf("Lock-Token: %s\n", lock->token); } locked = TRUE; locktokenfile = create_locktoken_file(dir->name, lock->token); } int ret = 0; DavResource *ls = dav_query(sn, "select D:getetag,idav:status,idav:split,`idav:content-hash`,idavprops:tags,idavprops:finfo,idavprops:xattributes,idavprops:link from / with depth = infinity"); if(!ls) { print_resource_error(sn, "/"); if(locked) { if(dav_unlock(root)) { print_resource_error(sn, "/"); } else { locked = FALSE; } } fprintf(stderr, "Abort\n"); dav_session_destroy(sn); // TODO: free return -1; } if(!ls->iscollection) { fprintf(stderr, "%s is not a collection.\nAbort.\n", ls->path); if(locked) { if(dav_unlock(root)) { print_resource_error(sn, "/"); } else { locked = FALSE; } } // TODO: free dav_session_destroy(sn); if(!locked && locktokenfile) { remove(locktokenfile); } return -1; } DavBool remove_file = cmd_getoption(a, "remove") ? 1 : 0; int sync_success = 0; int sync_delete = 0; int sync_error = 0; int sync_conflict = 0; UcxList *res_modified = NULL; UcxList *res_new = NULL; UcxList *res_moved = NULL; // type: MovedFile UcxList *res_link = NULL; UcxList *res_conflict = NULL; UcxList *res_mkdir = NULL; UcxList *res_metadata = NULL; UcxList *res_broken = NULL; UcxMap *lres_removed = ucx_map_new(16); // type: LocalResource* //UcxMap *svrres = ucx_map_new(db->resources->count); UcxMap *dbres = ucx_map_clone(db->resources, NULL, NULL); UcxList *stack = ucx_list_prepend(NULL, ls->children); while(stack) { DavResource *res = stack->data; stack = ucx_list_remove(stack, stack); while(res) { DavBool res_filtered = FALSE; if (res_matches_dir_filter(dir, res->path)) { res_filtered = TRUE; } else { UCX_FOREACH(elm, dir->filter.tags) { SyncTagFilter *tf = elm->data; if(!res_matches_tags(res, tf)) { res_filtered = TRUE; break; } } } if(res_filtered) { // don't delete files filtered by config localres_keep(db, res->path); res = res->next; continue; } if (!res_matches_tags(res, tagfilter)) { if(!remove_file) { localres_keep(db, res->path); } res = res->next; continue; } char *status = dav_get_string_property(res, "idav:status"); if(status && !strcmp(status, "broken")) { res = res->next; localres_keep(db, res->path); res_broken = ucx_list_append(res_broken, res); continue; } // check if a resource has changed on the server int change = resource_get_remote_change(a, res, dir, db); switch(change) { case REMOTE_NO_CHANGE: break; case REMOTE_CHANGE_MODIFIED: { res_modified = ucx_list_append(res_modified, res); break; } case REMOTE_CHANGE_NEW: { res_new = ucx_list_append(res_new, res); break; } case REMOTE_CHANGE_DELETED: break; // never happens case REMOTE_CHANGE_CONFLICT_LOCAL_MODIFIED: { res_conflict = ucx_list_append(res_conflict, res); break; } case REMOTE_CHANGE_METADATA: { res_metadata = ucx_list_append(res_metadata, res); break; } case REMOTE_CHANGE_MKDIR: { res_mkdir = ucx_list_append(res_mkdir, res); break; } case REMOTE_CHANGE_LINK: { res_link = ucx_list_append(res_link, res); break; } } // remove every server resource from dbres // all remaining elements are the resources that are removed // on the server ucx_map_cstr_remove(dbres, res->path); if(!dav_get_property_ns(res, DAV_NS, "split") && res->children) { stack = ucx_list_prepend(stack, res->children); } res = res->next; } } // find deleted resources // svrres currently contains all resources from the server // and will replace the current db->resources map later UcxMapIterator i = ucx_map_iterator(dbres); LocalResource *local; UCX_MAP_FOREACH(key, local, i) { if (res_matches_dir_filter(dir, local->path)) { continue; } if(!local->keep) { ucx_map_cstr_put(lres_removed, local->path, local); if(lres_removed->count > lres_removed->size * 2) { ucx_map_rehash(lres_removed); } } } // // BEGIN PULL // // the first thing we need are all directories to put the files in UCX_FOREACH(elm, res_mkdir) { DavResource *res = elm->data; if(sync_get_collection(a, dir, res, db)) { sync_error++; } } // we need a map for all conflicts for fast lookups UcxMap *conflicts = ucx_map_new(ucx_list_size(res_conflict)+16); UCX_FOREACH(elm, res_conflict) { DavResource *res = elm->data; ucx_map_cstr_put(conflicts, res->path, res); } if(SYNC_HASHING(dir)) { // check for moved/copied files UcxList *elm = res_new; UcxList *prev = NULL; UcxList *next = NULL; SYS_STAT s; for(;elm;elm=next) { DavResource *res = elm->data; prev = elm->prev; next = elm->next; if(dav_get_property_ns(res, DAV_PROPS_NS, "link")) { continue; } char *hash = sync_get_content_hash(res); if(!hash) { continue; } LocalResource *local = ucx_map_cstr_get(hashes, hash); if(!local) { continue; } char *local_path = util_concat_path(dir->path, local_resource_path(local)); int staterr = sys_stat(local_path, &s); free(local_path); if(staterr) { // origin doesn't exist or is inaccessible continue; } MovedFile *mf = malloc(sizeof(MovedFile)); mf->content = local; mf->resource = res; if(ucx_map_cstr_remove(lres_removed, local->path)) { mf->copy = FALSE; } else { mf->copy = TRUE; } res_moved = ucx_list_append(res_moved, mf); // remove item from res_new if(prev) { prev->next = next; } else { res_new = next; } if(next) { next->prev = prev; } } } // do copy/move operations UCX_FOREACH(elm, res_moved) { MovedFile *mf = elm->data; if(sync_shutdown) { break; } DavBool issplit = dav_get_property_ns(mf->resource, DAV_NS, "split") ? 1 : 0; if(ucx_map_cstr_get(conflicts, mf->resource->path)) { rename_conflict_file(dir, db, mf->resource->path, issplit); sync_conflict++; } // move file if(sync_move_resource(a, dir, mf->resource, mf->content, mf->copy, db, &sync_success)) { fprintf(stderr, "%s failed: %s\n", mf->copy?"copy":"move", mf->resource->path); sync_error++; } } // download all new, modified and conflict files UcxList *download = ucx_list_concat(res_modified, res_conflict); download = ucx_list_concat(res_new, download); download = ucx_list_concat(download, res_link); UCX_FOREACH(elm, download) { DavResource *res = elm->data; if(sync_shutdown) { break; } DavBool issplit = dav_get_property_ns(res, DAV_NS, "split") ? 1 : 0; if(ucx_map_cstr_get(conflicts, res->path)) { rename_conflict_file(dir, db, res->path, issplit); sync_conflict++; } // download the resource if(sync_get_resource(a, dir, res->path, res, db, &sync_success)) { fprintf(stderr, "resource download failed: %s\n", res->path); sync_error++; } } UCX_FOREACH(elm, res_metadata) { DavResource *res = elm->data; if(sync_shutdown) { break; } LocalResource *local = ucx_map_cstr_get(db->resources, res->path); if(local) { printf("update: %s\n", res->path); char *res_path = resource_local_path(res); char *local_path = create_local_path(dir, res->path); free(res_path); if(sync_store_metadata(dir, local_path, local, res)) { fprintf(stderr, "Metadata update failed: %s\n", res->path); sync_error++; } else { SYS_STAT s; if(sys_stat(local_path, &s)) { fprintf(stderr, "Cannot stat file after update: %s\n", strerror(errno)); } sync_set_metadata_from_stat(local, &s); sync_success++; } free(local_path); } else { // this should never happen but who knows fprintf(stderr, "Cannot update metadata of file %s: not in database\n", res->path); } } UcxList *rmdirs = NULL; UcxMapIterator mi = ucx_map_iterator(lres_removed); LocalResource *removed_res; UcxKey key; UCX_MAP_FOREACH(key, removed_res, mi) { if(sync_shutdown) { break; } int ret = sync_remove_local_resource(dir, removed_res); if(ret == -1) { rmdirs = ucx_list_append(rmdirs, removed_res); } else if(ret == 0) { LocalResource *local = ucx_map_cstr_remove(db->resources, removed_res->path); if(local) { local_resource_free(local); } sync_delete++; } } ucx_map_free(lres_removed); // sort dir list, we need to delete dirs with higher depth first rmdirs = ucx_list_sort(rmdirs, (cmp_func)resource_pathlen_cmp, NULL); // delete dirs UCX_FOREACH(elm, rmdirs) { LocalResource *local_dir = elm->data; if(!sync_remove_local_directory(dir, local_dir)) { // dir successfully removed, now remove the related db entry LocalResource *local = ucx_map_cstr_remove(db->resources, local_dir->path); if(local) { local_resource_free(local); } sync_delete++; } } // unlock repository if(locked) { if(dav_unlock(root)) { print_resource_error(sn, "/"); ret = -1; } else { locked = FALSE; } } // store db if(store_db(db, dir->database, dir->db_settings)) { fprintf(stderr, "Cannot store sync db\n"); ret = -2; } // cleanup dav_session_destroy(sn); if(!locked && locktokenfile) { remove(locktokenfile); } // Report if(ret != -2) { char *str_success = sync_success == 1 ? "file" : "files"; char *str_delete = sync_delete == 1 ? "file" : "files"; char *str_error = sync_error == 1 ? "error" : "errors"; char *str_conflict = sync_conflict == 1 ? "conflict" : "conflicts"; printf("Result: %d %s pulled, %d %s deleted, %d %s, %d %s\n", sync_success, str_success, sync_delete,str_delete, sync_conflict, str_conflict, sync_error, str_error); } return ret; } RemoteChangeType resource_get_remote_change( CmdArgs *a, DavResource *res, SyncDirectory *dir, SyncDatabase *db) { char *etag = dav_get_string_property(res, "D:getetag"); if(!etag) { fprintf(stderr, "Error: resource %s has no etag\n", res->path); return REMOTE_NO_CHANGE; } char *hash = sync_get_content_hash(res); DavBool issplit = dav_get_property(res, "idav:split") ? TRUE : FALSE; if(issplit) { util_remove_trailing_pathseparator(res->path); } DavBool iscollection = res->iscollection && !issplit; RemoteChangeType type = cmd_getoption(a, "conflict") ? REMOTE_CHANGE_MODIFIED : REMOTE_CHANGE_CONFLICT_LOCAL_MODIFIED; LocalResource *local = ucx_map_cstr_get(db->resources, res->path); char *local_path = create_local_path(dir, res->path); char *link = SYNC_SYMLINK(dir) ? dav_get_string_property_ns(res, DAV_PROPS_NS, "link") : NULL; SYS_STAT s; DavBool exists = 1; if(sys_stat(local_path, &s)) { if(errno != ENOENT) { fprintf(stderr, "Cannot stat file: %s\n", local_path); free(local_path); return REMOTE_NO_CHANGE; } exists = 0; } RemoteChangeType ret = REMOTE_NO_CHANGE; if(iscollection) { if(!exists) { ret = REMOTE_CHANGE_MKDIR; } else if(local && S_ISDIR(s.st_mode)) { local->isdirectory = 1; // make sure isdirectory is set } else { // set change to REMOTE_CHANGE_MKDIR, which will fail later ret = REMOTE_CHANGE_MKDIR; } } else if(local) { DavBool nochange = FALSE; if(SYNC_SYMLINK(dir) && nullstrcmp(link, local->link_target)) { ret = REMOTE_CHANGE_LINK; nochange = TRUE; } else if(local->etag) { sstr_t e = sstr(etag); if(sstrprefix(e, S("W/"))) { e = sstrsubs(e, 2); } if(!strcmp(e.ptr, local->etag)) { // resource is already up-to-date on the client nochange = TRUE; } } if(!nochange) { if(!(exists && s.st_mtime != local->last_modified)) { type = REMOTE_CHANGE_MODIFIED; } ret = type; } } else if(link) { if(type == REMOTE_CHANGE_CONFLICT_LOCAL_MODIFIED) { ret = REMOTE_CHANGE_CONFLICT_LOCAL_MODIFIED; } else { ret = REMOTE_CHANGE_LINK; } } else if(exists) { ret = type; } else { ret = REMOTE_CHANGE_NEW; } // if hashing is enabled we can compare the hash of the remote file // with the local file to test if a file is really modified DavBool update_db = FALSE; char *update_hash = NULL; if (!iscollection && !link && (ret == REMOTE_CHANGE_MODIFIED || ret == REMOTE_CHANGE_CONFLICT_LOCAL_MODIFIED) && exists && hash && !dir->pull_skip_hashing) { // because rehashing a file is slow, there is a config element for // disabling this (pull-skip-hashing) char *local_hash = util_file_hash(local_path); if(local_hash) { if(!strcmp(hash, local_hash)) { ret = REMOTE_NO_CHANGE; update_db = TRUE; update_hash = local_hash; // if local already exists, update the hash here // because it is possible that there are metadata updates // and in this case the db will updated later and needs // the current hash if(local) { if(local->hash) { free(local->hash); } local->hash = local_hash; } } else { free(local_hash); } } } // if a file is not modified, check if the metadata has changed while(ret == REMOTE_NO_CHANGE && local) { // check if tags have changed if(dir->tagconfig) { DavXmlNode *tagsprop = dav_get_property_ns(res, DAV_PROPS_NS, "tags"); UcxList *remote_tags = NULL; if(tagsprop) { remote_tags = parse_dav_xml_taglist(tagsprop); } char *remote_hash = create_tags_hash(remote_tags); if(nullstrcmp(remote_hash, local->remote_tags_hash)) { ret = REMOTE_CHANGE_METADATA; } if(remote_hash) { free(remote_hash); } free_taglist(remote_tags); if(ret == REMOTE_CHANGE_METADATA) { break; } } // check if extended attributes have changed if((dir->metadata & FINFO_XATTR) == FINFO_XATTR) { DavXmlNode *xattr = dav_get_property_ns(res, DAV_PROPS_NS, "xattributes"); char *xattr_hash = get_xattr_hash(xattr); if(nullstrcmp(xattr_hash, local->xattr_hash)) { ret = REMOTE_CHANGE_METADATA; break; } } // check if finfo has changed DavXmlNode *finfo = dav_get_property_ns(res, DAV_PROPS_NS, "finfo"); if((dir->metadata & FINFO_MODE) == FINFO_MODE) { FileInfo f; finfo_get_values(finfo, &f); if(f.mode_set && f.mode != local->mode) { ret = REMOTE_CHANGE_METADATA; break; } } break; } // if update_db is set, a file was modified on the server, but we already // have the file content, but we need to update the db if(ret == REMOTE_NO_CHANGE && update_db) { if(!local) { local = calloc(1, sizeof(LocalResource)); local->path = strdup(res->path); ucx_map_cstr_put(db->resources, local->path, local); } // update local res SYS_STAT s; if(!sys_stat(local_path, &s)) { sync_set_metadata_from_stat(local, &s); } else { fprintf(stderr, "stat failed for file: %s : %s", local_path, strerror(errno)); } local_resource_set_etag(local, etag); if(!local->hash) { local->hash = update_hash; } // else: hash already updated } free(local_path); return ret; } void sync_set_metadata_from_stat(LocalResource *local, SYS_STAT *s) { local->last_modified = s->st_mtime; local->mode = s->st_mode & 07777; local->uid = s->st_uid; local->gid = s->st_gid; local->size = s->st_size; } static UcxList* sync_download_changed_parts( DavResource *res, LocalResource *local, FILE *out, size_t blocksize, uint64_t *blockcount, int64_t *truncate_file, int *err) { UcxList *updates = NULL; size_t local_numparts = local ? local->numparts : 0; fseeko(out, 0, SEEK_END); off_t end = ftello(out); int error = 0; UcxBuffer *buf = ucx_buffer_new(NULL, blocksize, 0); int64_t maxsize = -1; DavResource *part = res->children; uint64_t i = 0; while(part) { char *res_name = part->name; while(res_name[0] == '0' && res_name[1] != '\0') { res_name++; } uint64_t partnum = 0; if(util_strtouint(res_name, &partnum)) { DavBool download_part = FALSE; char *etag = dav_get_string_property_ns(part, "DAV:", "getetag"); if(partnum >= local_numparts) { // new part download_part = TRUE; } else { FilePart p = local->parts[partnum]; // local is always non-null here if(etag) { if(nullstrcmp(etag, p.etag)) { download_part = TRUE; } } } uint64_t offset; int mul_err = util_uint_mul(partnum, blocksize, &offset); if(mul_err || offset >= INT64_MAX) { error = 1; fprintf(stderr, "Error: part number too high\n"); break; } int64_t block_end = 0; if(download_part) { if(fseeko(out, offset, SEEK_SET)) { error = 1; fprintf(stderr, "Error: fseek failed: %s\n", strerror(errno)); break; } buf->pos = 0; buf->size = 0; if(dav_get_content(part, buf,(dav_write_func)ucx_buffer_write)) { fprintf(stderr, "Error: cannot download part: %s\n", part->name); error = 1; break; } if(fwrite(buf->space, 1, buf->size, out) == 0) { perror("write"); error = 1; break; } FilePart *update = calloc(1, sizeof(FilePart)); update->block = partnum; update->etag = etag ? strdup(etag) : NULL; update->hash = dav_create_hash(buf->space, buf->size); updates = ucx_list_append(updates, update); block_end = offset+buf->size; } else { if(offset+blocksize > end) { // if we don't download the block, we don't know the size // but it can't be bigger than the file block_end = end; } else { block_end = offset+blocksize; } } if(block_end > maxsize) { maxsize = block_end; } i++; } // else: res is not a regular file part part = part->next; } ucx_buffer_free(buf); if(error) { *err = 1; ucx_list_free_content(updates, (ucx_destructor)filepart_free); return NULL; } if(maxsize < end) { *truncate_file = maxsize; } else { *truncate_file = -1; } *err = 0; *blockcount = i; return updates; } int copy_file(const char *from, const char *to) { FILE *in = sys_fopen(from, "rb"); if(!in) { return 1; } FILE *out = sys_fopen(to, "wb"); if(!out) { fclose(in); return 1; } ucx_stream_copy(in, out, (read_func)fread, (write_func)fwrite); fclose(in); fclose(out); return 0; } typedef int (*renamefunc)(const char*,const char*); int sync_move_resource( CmdArgs *a, SyncDirectory *dir, DavResource *res, LocalResource *content, DavBool copy, SyncDatabase *db, int *counter) { renamefunc fn = copy ? copy_file : sys_rename; char *new_path = util_concat_path(dir->path, res->path); char *old_path = util_concat_path(dir->path, local_resource_path(content)); printf("%s: %s -> %s\n", copy?"copy":"move", content->path, res->path); if(fn(old_path, new_path)) { free(new_path); free(old_path); return 1; } (*counter)++; char *etag = dav_get_string_property(res, "D:getetag"); char *content_hash = sync_get_content_hash(res); LocalResource *local = NULL; if(copy) { local = local_resource_copy(content, res->path); } else { // reuse previous LocalResource (content) // remove it from db->resources, change path and put it back local = ucx_map_cstr_remove(db->resources, content->path); if(!local) { // can't happen, but handle it nevertheless local = content; } free(content->path); local->path = strdup(res->path); } ucx_map_cstr_put(db->resources, local->path, local); if(sync_store_metadata(dir, new_path, local, res)) { fprintf(stderr, "Cannot store metadata: %s\n", res->path); } // dont free local->etag, because local_resource_set_etag will do that if(local->hash) { free(local->hash); } SYS_STAT s; if(sys_stat(new_path, &s)) { fprintf(stderr, "Cannot stat file %s: %s\n", new_path, strerror(errno)); } // set metadata from stat local_resource_set_etag(local, etag); local->hash = nullstrdup(content_hash); sync_set_metadata_from_stat(local, &s); local->skipped = FALSE; return 0; } int sync_get_resource( CmdArgs *a, SyncDirectory *dir, const char *path, DavResource *res, SyncDatabase *db, int *counter) { char *link = SYNC_SYMLINK(dir) ? dav_get_string_property_ns(res, DAV_PROPS_NS, "link") : NULL; LocalResource *local = ucx_map_cstr_get(db->resources, path); char *local_path; if(link) { char *res_path = resource_local_path(res); local_path = create_local_path(dir, res_path); free(res_path); } else { local_path = create_local_path(dir, path); } char *etag = dav_get_string_property(res, "D:getetag"); SYS_STAT s; memset(&s, 0, sizeof(SYS_STAT)); char *blocksize_str = dav_get_string_property_ns(res, DAV_NS, "split"); uint64_t blocksize = 0; DavBool issplit = FALSE; if(blocksize_str && !link) { if(!util_strtouint(blocksize_str, &blocksize)) { fprintf(stderr, "Error: split property does not contain an integer.\n"); return 1; } issplit = TRUE; } UcxList *part_updates = NULL; uint64_t blockcount = 0; char *content_hash = NULL; if(res->iscollection && !issplit) { // why are we here? return 0; } int ret = 0; char *tmp_path = NULL; FILE *out = NULL; if(!link) { if(!issplit) { tmp_path = create_tmp_download_path(local_path); if(!tmp_path) { fprintf(stderr, "Cannot create tmp path for %s\n", local_path); free(local_path); return -1; } out = sys_fopen(tmp_path , "wb"); } else { out = sys_fopen(local_path, "r+b"); if(!out && errno == ENOENT) { out = sys_fopen(local_path, "wb"); } } if(!out) { fprintf(stderr, "Cannot open output file: %s\n", local_path); free(local_path); if(tmp_path) { free(tmp_path); } return -1; } } int64_t truncate_file = -1; if(!link) { printf("get: %s\n", path); if(issplit) { part_updates = sync_download_changed_parts(res, local, out, blocksize, &blockcount, &truncate_file, &ret); } else { ret = dav_get_content(res, out, (dav_write_func)fwrite); } fclose(out); } else { printf("link: %s -> %s\n", path, link); if(sys_symlink(link, local_path)) { perror("symlink"); ret = 1; } } if(issplit || (SYNC_HASHING(dir) && !link)) { if(truncate_file >= 0) { // only true if issplit is true if(truncate(local_path, truncate_file)) { perror("truncate"); } } char *res_hash = sync_get_content_hash(res); if(res_hash) { content_hash = res_hash; } else { content_hash = util_file_hash(local_path); } } if(ret == 0) { (*counter)++; if(tmp_path) { if(dir->trash && dir->backuppull) { move_to_trash(dir, local_path); } if(sys_rename(tmp_path, local_path)) { fprintf( stderr, "Cannot rename file %s to %s\n", tmp_path, local_path); perror(""); free(tmp_path); free(local_path); return -1; } } if(!local) { // new local resource local = calloc(1, sizeof(LocalResource)); local->path = strdup(path); ucx_map_cstr_put(db->resources, local->path, local); } if(sync_store_metadata(dir, local_path, local, res)) { fprintf(stderr, "Cannot store metadata: %s\n", path); } if(local->etag) { free(local->etag); local->etag = NULL; } if(local->hash) { free(local->hash); local->hash = NULL; } if(local->link_target) { free(local->link_target); local->link_target = NULL; } stat_func statfn; if(link) { local->link_target = strdup(link); statfn = sys_lstat; } else { statfn = sys_stat; } update_parts(local, part_updates, blockcount); if(statfn(local_path, &s)) { fprintf(stderr, "Cannot stat file %s: %s\n", local_path, strerror(errno)); } // set metadata from stat local_resource_set_etag(local, etag); if(content_hash) { local->hash = content_hash; } sync_set_metadata_from_stat(local, &s); local->skipped = FALSE; } else if(tmp_path) { if(sys_unlink(tmp_path)) { fprintf(stderr, "Cannot remove tmp file: %s\n", tmp_path); } } if(tmp_path) { free(tmp_path); } free(local_path); return ret; } int sync_get_collection( CmdArgs *a, SyncDirectory *dir, DavResource *res, SyncDatabase *db) { char *res_path = resource_local_path(res); char *local_path = create_local_path(dir, res->path); free(res_path); printf("get: %s\n", res->path); // create directory // ignore error if it already exists if(sys_mkdir(local_path) && errno != EEXIST) { fprintf(stderr, "Cannot create directory %s: %s", local_path, strerror(errno)); free(local_path); return 1; } // stat for getting metadata SYS_STAT s; if(sys_stat(local_path, &s)) { fprintf(stderr, "Cannot stat directory %s: %s", local_path, strerror(errno)); free(local_path); return 1; } // if it doesn't exist in the db, create an entry for the dir LocalResource *local = ucx_map_cstr_get(db->resources, res->path); if(!local) { local = calloc(1, sizeof(LocalResource)); local->path = strdup(res->path); ucx_map_cstr_put(db->resources, local->path, local); } local->isdirectory = 1; // cleanup LocalResource if(local->etag) { free(local->etag); local->etag = NULL; } if(local->hash) { free(local->hash); local->hash = NULL; } if(local->link_target) { free(local->link_target); local->link_target = NULL; } local->skipped = 0; local->size = 0; // set metadata if(sync_store_metadata(dir, local_path, local, res)) { fprintf(stderr, "Cannot store metadata: %s\n", res->path); } sync_set_metadata_from_stat(local, &s); // cleanup free(local_path); return 0; } int sync_remove_local_resource(SyncDirectory *dir, LocalResource *res) { char *local_path = create_local_path(dir, res->path); SYS_STAT s; if(sys_stat(local_path, &s)) { free(local_path); return -2; } if(S_ISDIR(s.st_mode)) { free(local_path); return -1; } if(s.st_mtime != res->last_modified) { free(local_path); return -2; } printf("delete: %s\n", res->path); int ret = 0; if(dir->trash) { move_to_trash(dir, local_path); } else if(sys_unlink(local_path)) { fprintf(stderr, "Cannot remove file %s\n", local_path); ret = -2; } free(local_path); return ret; } int sync_remove_local_directory(SyncDirectory *dir, LocalResource *res) { int ret = 0; char *local_path = create_local_path(dir, res->path); printf("delete: %s\n", res->path); if(rmdir(local_path)) { // don't print error when dirs are not empty // because that can regulary happen, because the pull conflict // detection can prevent files from being deleted and in this case // the parent dir is not empty if(errno != ENOTEMPTY) { fprintf(stderr, "rmdir: %s : %s\n", local_path, strerror(errno)); } ret = 1; } free(local_path); return ret; } void rename_conflict_file(SyncDirectory *dir, SyncDatabase *db, char *path, DavBool copy) { char *local_path = create_local_path(dir, path); char *parent = util_parent_path(local_path); renamefunc fn = copy ? copy_file : sys_rename; int rev = 0; SYS_STAT s; int loop = 1; do { char *res_parent = util_parent_path(path); char *res_name = util_resource_name(path); sstr_t new_path = ucx_sprintf( "%sorig.%d.%s", parent, rev, res_name); sstr_t new_res_path = ucx_sprintf( "%sorig.%d.%s", res_parent, rev, res_name); if(sys_stat(new_path.ptr, &s)) { if(errno == ENOENT) { loop = 0; printf("conflict: %s\n", local_path); if(fn(local_path, new_path.ptr)) { //printf("errno: %d\n", errno); fprintf( stderr, "Cannot rename file %s to %s\n", local_path, new_path.ptr); } else { LocalResource *conflict = calloc(1, sizeof(LocalResource)); conflict->path = strdup(new_res_path.ptr); conflict->conflict_source = strdup(path); ucx_map_cstr_put(db->conflict, new_res_path.ptr, conflict); } } } rev++; free(res_parent); free(new_path.ptr); free(new_res_path.ptr); } while(loop); free(parent); free(local_path); } char* create_tmp_download_path(char *path) { char *new_path = NULL; char *parent = util_parent_path(path); for (int i=0;;i++) { sstr_t np = ucx_asprintf( ucx_default_allocator(), "%sdownload%d-%s", parent, i, util_resource_name(path)); SYS_STAT s; if(sys_stat(np.ptr, &s)) { if(errno == ENOENT) { new_path = np.ptr; } break; } free(np.ptr); }; free(parent); return new_path; } void move_to_trash(SyncDirectory *dir, char *path) { char *new_path = NULL; for (int i=0;;i++) { sstr_t np = ucx_asprintf( ucx_default_allocator(), "%s%d-%s", dir->trash, i, util_resource_name(path)); SYS_STAT s; if(sys_stat(np.ptr, &s)) { if(errno == ENOENT) { new_path = np.ptr; } break; } free(np.ptr); }; if(!new_path) { fprintf(stderr, "Cannot move file %s to trash.\n", path); return; } if(sys_rename(path, new_path)) { //printf("errno: %d\n", errno); fprintf( stderr, "Cannot rename file %s to %s\n", path, new_path); } free(new_path); } static int res_isconflict(SyncDatabase *db, LocalResource *res) { return ucx_map_cstr_get(db->conflict, res->path) ? 1 : 0; } int cmd_push(CmdArgs *a, DavBool archive) { if(a->argc != 1) { fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many"); return -1; } // if there are syntax errors in the command line, fail asap. SyncTagFilter* tagfilter = parse_tagfilter_string( cmd_getoption(a, "tags"), DAV_SYNC_TAGFILTER_SCOPE_RESOURCE); if (!tagfilter) { fprintf(stderr, "Malformed tag filter\n"); return -1; } SyncDirectory *dir = scfg_get_dir(a->argv[0]); if(!dir) { fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]); return -1; } if(scfg_check_dir(dir)) { return -1; } if(cmd_getoption(a, "snapshot")) { if(dir->versioning) { dir->versioning->always = TRUE; } else { fprintf(stderr, "Error: versioning not configured for the sync directory\nAbort.\n"); return -1; } } int cmd = archive ? SYNC_CMD_ARCHIVE : SYNC_CMD_PUSH; if((dir->allow_cmd & cmd) != cmd) { fprintf(stderr, "Command '%s' is not allowed for this sync dir\n", archive ? "archive" : "push"); print_allowed_cmds(dir); return -1; } Repository *repo = get_repository(sstr(dir->repository)); if(!repo) { fprintf(stderr, "Unkown repository %s\n", dir->name); return -1; } SyncDatabase *db = load_db(dir->database); if(!db) { fprintf(stderr, "Cannot load database file: %s\n", dir->database); return -1; } remove_deleted_conflicts(dir, db); DavSession *sn = create_session(ctx, repo, dir->collection); ucx_mempool_reg_destr(sn->mp, db, (ucx_destructor)destroy_db); if (cmd_getoption(a, "verbose")) { curl_easy_setopt(sn->handle, CURLOPT_VERBOSE, 1L); curl_easy_setopt(sn->handle, CURLOPT_STDERR, stderr); } if(SYNC_STORE_HASH(dir)) { sn->flags |= DAV_SESSION_STORE_HASH; } DavBool restore_removed = cmd_getoption(a, "restore-removed") ? 1 : 0; DavBool restore_modified = cmd_getoption(a, "restore-modified") ? 1 : 0; DavBool restore = restore_removed || restore_modified; int depth = restore ? -1 : 0; DavResource *root = dav_query(sn, "select D:getetag,idav:status from / with depth = %d", depth); if(!root) { print_resource_error(sn, "/"); dav_session_destroy(sn); fprintf(stderr, "Abort\n"); return -1; } UcxMap *svrres = NULL; if(restore) { svrres = ucx_map_new(1024); res2map(root, svrres); } int cdt = cmd_getoption(a, "conflict") ? 0 : 1; // conflict detection // lock repository DavBool locked = FALSE; char *locktokenfile = NULL; if((dir->lockpush || cmd_getoption(a, "lock")) && !cmd_getoption(a, "nolock")) { if(dav_lock_t(root, dir->lock_timeout)) { print_resource_error(sn, "/"); dav_session_destroy(sn); fprintf(stderr, "Abort\n"); return -1; } DavLock *lock = dav_get_lock(sn, "/"); if(lock) { printf("Lock-Token: %s\n", lock->token); } locked = TRUE; locktokenfile = create_locktoken_file(dir->name, lock->token); } DavBool remove_file = cmd_getoption(a, "remove") ? 1 : 0; UcxMap *db_hashes = NULL; if(SYNC_HASHING(dir)) { db_hashes = create_hash_index(db); } int sync_success = 0; int sync_delete = 0; int sync_conflict = 0; int sync_error = 0; UcxList *ls_new = NULL; UcxList *ls_modified = NULL; UcxList *ls_conflict = NULL; UcxList *ls_update = NULL; UcxList *ls_delete = NULL; UcxList *ls_move = NULL; UcxList *ls_copy = NULL; UcxList *ls_mkcol = NULL; // upload all changed files //UcxList *resources = cmd_getoption(a, "read") ? // read_changes(dir, db) : local_scan(dir, db); UcxList *resources = local_scan(dir, db); UcxMap *resources_map = ucx_map_new(ucx_list_size(resources)+16); UCX_FOREACH(elm, resources) { LocalResource *local_res = elm->data; // ignore all files, that are excluded by a static filter (sync.xml) // static include/exclude filter if(res_matches_dir_filter(dir, local_res->path+1)) { continue; } // static tag filter UCX_FOREACH(elm, dir->filter.tags) { SyncTagFilter *tf = elm->data; if(!localres_matches_tags(dir, local_res, tf)) { continue; } } // we need a fast file lookup map later to detect deleted files ucx_map_cstr_put(resources_map, local_res->path, local_res); // dynamic tag filter if(!localres_matches_tags(dir, local_res, tagfilter)) { if(!remove_file) { LocalResource *dbres = ucx_map_cstr_get( db->resources, local_res->path); if(dbres) { // this makes sure the file will not be deleted later dbres->keep = TRUE; } } continue; } // skip conflict backups silently if(res_isconflict(db, local_res)) { ls_conflict = ucx_list_append(ls_conflict, local_res); continue; } int is_changed = local_resource_is_changed( dir, db, local_res, svrres, restore_removed, restore_modified); if(is_changed) { if(local_res->isdirectory) { ls_mkcol = ucx_list_append(ls_mkcol, local_res); } else if(local_res->isnew) { ls_new = ucx_list_append(ls_new, local_res); } else { ls_modified = ucx_list_append(ls_modified, local_res); } } else if(local_res->metadata_updated) { ls_update = ucx_list_append(ls_update, local_res); } if(local_res->isnew) { if(local_resource_load_metadata(dir, local_res)) { fprintf( stderr, "Failed to load metadata: %s\n", local_resource_path(local_res)); } } } if(SYNC_STORE_HASH(dir)) { // calculate hashes of all new files and check if a file // was moved or is a copy UcxList *elm = ls_new; while(elm) { LocalResource *local = elm->data; UcxList *prev = elm->prev; UcxList *next = elm->next; if(local->isdirectory || local->link_target) { elm = elm->next; continue; } char *local_path = util_concat_path(dir->path, local_resource_path(local)); char *hash = util_file_hash(local_path); local->hash = hash; // check if a file with this hash already exists LocalResource *origin = ucx_map_cstr_get(db_hashes, hash); if(origin) { local->origin = local_resource_copy(origin, origin->path); // the file is a copied/moved file // check if the file is in the resources_map, because then // it still exists if(ucx_map_cstr_get(resources_map, origin->path)) { ls_copy = ucx_list_append(ls_copy, local); } else { ls_move = ucx_list_append(ls_move, local); // put file in resources_map to prevent deletion ucx_map_cstr_put(resources_map, origin->path, local); } // remove list elemend from ls_new if(prev) { prev->next = next; } else { ls_new = next; } if(next) { next->prev = prev; } } free(local_path); elm = next; } } // find all deleted files and cleanup the database UcxMapIterator i = ucx_map_iterator(db->resources); LocalResource *local; UcxList *removed_res = NULL; UCX_MAP_FOREACH(key, local, i) { // all filtered files should be removed from the database if(res_matches_dir_filter(dir, local->path+1)) { ucx_map_cstr_remove(db->resources, local->path); continue; } UCX_FOREACH(elm, dir->filter.tags) { SyncTagFilter *tf = elm->data; if(!localres_matches_tags(dir, local, tf)) { ucx_map_cstr_remove(db->resources, local->path); continue; } } if(!ucx_map_get(resources_map, key)) { // The current LocalResource is in the database but doesn't exist // in the filesystem anymore. This means the file was deleted // and should be deleted on the server if(!archive) { ls_delete = ucx_list_append(ls_delete, local); } else { removed_res = ucx_list_prepend(removed_res, local); } } } UCX_FOREACH(elm, removed_res) { LocalResource *local = elm->data; ucx_map_cstr_remove(db->resources, local->path); } // // BEGIN PUSH // int ret = 0; int error = 0; // create collections for(UcxList *elm=ls_mkcol;elm && !sync_shutdown;elm=elm->next) { LocalResource *local_res = elm->data; DavResource *res = dav_resource_new(sn, local_res->path); if(!res) { print_resource_error(sn, local_res->path); ret = -1; sync_error++; } int abort = 0; dav_exists(res); if(sn->error == DAV_NOT_FOUND) { // create collection // TODO: show 405 printf("mkcol: %s\n", local_res->path); if(sync_mkdir(dir, res, local_res) && sn->error != DAV_METHOD_NOT_ALLOWED) { print_resource_error(sn, res->path); ret = -1; sync_error++; error = 1; abort = 1; } } else if(sn->error != DAV_OK) { // dav_exists() failed print_resource_error(sn, local_res->path); ret = -1; sync_error++; error = 1; abort = 1; } if(!abort) { // success if(local_res->metadata_updated) { sync_update_metadata(dir, sn, res, local_res); } // remove old db entry (if it exists) // and add add new entry LocalResource *dbres = ucx_map_cstr_remove(db->resources, local_res->path); ucx_map_cstr_put(db->resources, local_res->path, local_res); } dav_resource_free(res); } // copy/move files DavBool copy = TRUE; if(!ls_copy) { copy = FALSE; ls_copy = ls_move; } for(UcxList *elm=ls_copy;elm && !sync_shutdown;elm=elm->next) { LocalResource *local = elm->data; int err = 0; DavResource *res = dav_resource_new(sn, local->path); if(dav_exists(res)) { printf("conflict: %s\n", local->path); local->last_modified = 0; nullfree(local->etag); local->etag = NULL; nullfree(local->hash); local->hash = NULL; local->skipped = TRUE; sync_conflict++; } else { DavResource *origin_res = dav_resource_new(sn, local->origin->path); int origin_changed = remote_resource_is_changed( sn, dir, db, origin_res, local->origin, NULL); if(origin_changed) { // upload with put ls_modified = ucx_list_prepend(ls_modified, local); } else { printf("%s: %s -> %s\n", copy ? "copy":"move", local->origin->path, local->path); err = sync_move_remote_resource( dir, db, origin_res, local, copy, &sync_success); } } if(err) { sync_error++; print_resource_error(sn, res->path); ret = -1; error = 1; } else { LocalResource *dbres = ucx_map_cstr_remove(db->resources, local->path); ucx_map_cstr_put(db->resources, local->path, local); } if(copy && !elm->next) { // finished copy, begin move elm->next = ls_move; copy = FALSE; } } // upload changed files ls_modified = ucx_list_concat(ls_new, ls_modified); for(UcxList *elm=ls_modified;elm && !sync_shutdown;elm=elm->next) { LocalResource *local_res = elm->data; int err = 0; DavResource *res = dav_resource_new(sn, local_res->path); if(!res) { print_resource_error(sn, local_res->path); ret = -1; sync_error++; } else { DavBool equal = FALSE; int changed = remote_resource_is_changed(sn, dir, db, res, local_res, &equal); if(equal) { char *etag = dav_get_string_property(res, "D:getetag"); if(local_res->metadata_updated) { ls_update = ucx_list_prepend(ls_update, local_res); } else if(etag) { // update etag in db if(local_res->etag) { free(local_res->etag); } local_res->etag = strdup(etag); } } else if(cdt && changed) { printf("conflict: %s\n", local_res->path); local_res->last_modified = 0; nullfree(local_res->etag); local_res->etag = NULL; nullfree(local_res->hash); local_res->hash = NULL; local_res->skipped = TRUE; sync_conflict++; } else { if(local_res->link_target) { printf( "link: %s -> %s\n", local_res->path, local_res->link_target); } else { printf("put: %s\n", local_res->path); } if(sync_put_resource(dir, res, local_res, &sync_success)) { sync_error++; print_resource_error(sn, res->path); ret = -1; error = 1; err = 1; } } if(!err) { LocalResource *dbres = ucx_map_cstr_remove(db->resources, local_res->path); ucx_map_cstr_put(db->resources, local_res->path, local_res); } } dav_resource_free(res); } // metadata updates for(UcxList *elm=ls_update;elm && !sync_shutdown;elm=elm->next) { LocalResource *local_res = elm->data; DavResource *res = dav_resource_new(sn, local_res->path); if(local_res->metadata_updated) { printf("update: %s\n", local_res->path); if(!sync_update_metadata(dir, sn, res, local_res)) { LocalResource *dbres = ucx_map_cstr_remove(db->resources, local_res->path); ucx_map_cstr_put(db->resources, local_res->path, local_res); } } } // delete all removed files ls_delete = ucx_list_sort(ls_delete, (cmp_func)resource_pathlen_cmp, NULL); UcxList *cols = NULL; UcxList **col_list = &cols; UcxList *deletelist = ls_delete; for(int i=0;i<2;i++) { // the first iteration deletes everything from ls_delete except // all collections, which are stored in cols // in the second run all collections will be deleted for(UcxList *elm=deletelist;elm && !sync_shutdown;elm=elm->next) { LocalResource *local = elm->data; if(local->keep) { continue; } if(sync_delete_remote_resource(dir, sn, local, &sync_delete, col_list)) { if(sn->error != DAV_NOT_FOUND) { print_resource_error(sn, local->path); sync_error++; break; } } else { LocalResource *dbres = ucx_map_cstr_remove(db->resources, local->path); //local_resource_free(dbres); } } deletelist = cols; col_list = NULL; } // unlock repository if(locked) { if(dav_unlock(root)) { print_resource_error(sn, "/"); ret = -1; } else { locked = FALSE; } } // store db if(store_db(db, dir->database, dir->db_settings)) { fprintf(stderr, "Cannot store sync db\n"); ret = -2; } // cleanup if(!locked && locktokenfile) { remove(locktokenfile); } dav_session_destroy(sn); // Report if(ret != -2) { char *str_success = sync_success == 1 ? "file" : "files"; char *str_delete = sync_delete == 1 ? "file" : "files"; char *str_conflict = sync_conflict == 1 ? "conflict" : "conflicts"; char *str_error = sync_error == 1 ? "error" : "errors"; printf("Result: %d %s pushed, ", sync_success, str_success); if(!archive) { printf("%d %s deleted, ", sync_delete, str_delete); } printf("%d %s, %d %s\n", sync_conflict, str_conflict, sync_error, str_error); } return ret; } static int localres_cmp_path(LocalResource *a, LocalResource *b, void *n) { return strcmp(a->path, b->path); } int cmd_restore(CmdArgs *a) { char *syncdir = cmd_getoption(a, "syncdir"); if(!syncdir && a->argc == 0) { fprintf(stderr, "No syncdir or files specified\n"); return -1; } char *version = cmd_getoption(a, "version"); if(version) { if(a->argc != 1) { fprintf(stderr, "If the -V option is enabled, only one file can be specified\n"); return -1; } } SyncDirectory *dir = NULL; UcxMap *files = NULL; if(syncdir) { dir = scfg_get_dir(syncdir); } LocalResource nres; if(a->argc > 0) { files = ucx_map_new(a->argc+8); // get all specified files and check the syncdir SyncDirectory *sd = NULL; for(int i=0;i<a->argc;i++) { SyncFile f; int err = sync_get_file(a, a->argv[i], &f, FALSE); if(err) { sync_print_get_file_err(a->argv[i], err); return 1; } if(!sd) { sd = f.dir; } else { if(f.dir != sd) { fprintf(stderr, "Not all files are in the same syncdir\n"); return 1; } } ucx_map_cstr_put(files, f.path, &nres); } dir = sd; } if(!dir) { fprintf(stderr, "Unknown sync dir: %s\n", syncdir); return -1; } if(scfg_check_dir(dir)) { return -1; } if((dir->allow_cmd & SYNC_CMD_RESTORE) != SYNC_CMD_RESTORE) { fprintf(stderr, "Command ''restore'' is not allowed for this sync dir\n"); print_allowed_cmds(dir); return -1; } DavBool restore_modified = cmd_getoption(a, "restore-modified") ? 1 : 0; DavBool restore_removed = cmd_getoption(a, "restore-removed") ? 1 : 0; if(!restore_modified && !restore_removed) { restore_modified = 1; restore_removed = 1; } SyncDatabase *db = load_db(dir->database); if(!db) { fprintf(stderr, "Cannot load database file: %s\n", dir->database); return -1; } remove_deleted_conflicts(dir, db); UcxList *modified = NULL; UcxList *deleted = NULL; // iterate over all db resources and check if any resource is // modified or deleted UcxMapIterator i = ucx_map_iterator(files ? files : db->resources); LocalResource *resource; UCX_MAP_FOREACH(key, resource, i) { if(resource == &nres) { resource = ucx_map_get(db->resources, key); if(!resource) { continue; } } char *file_path = create_local_path(dir, resource->path); SYS_STAT s; if(sys_stat(file_path, &s)) { if(errno == ENOENT) { if(restore_removed) { deleted = ucx_list_prepend(deleted, resource); } } else { fprintf(stderr, "Cannot stat file: %s\n", file_path); perror(""); } } else { if(files) { modified = ucx_list_prepend(modified, resource); } else if(!resource->isdirectory && !S_ISDIR(s.st_mode)) { if(resource->last_modified != s.st_mtime || resource->size != s.st_size) { if(restore_modified) { modified = ucx_list_prepend(modified, resource); } } } } free(file_path); } if(files) { ucx_map_free(files); } int ret = 0; // create DavSession Repository *repo = get_repository(sstr(dir->repository)); if(!repo) { fprintf(stderr, "Unkown repository %s\n", dir->name); return -1; } DavSession *sn = create_session(ctx, repo, dir->collection); ucx_mempool_reg_destr(sn->mp, db, (ucx_destructor)destroy_db); if (cmd_getoption(a, "verbose")) { curl_easy_setopt(sn->handle, CURLOPT_VERBOSE, 1L); curl_easy_setopt(sn->handle, CURLOPT_STDERR, stderr); } // lock repository char *locktokenfile = NULL; DavBool locked = FALSE; DavResource *root = dav_resource_new(sn, "/"); root->iscollection = TRUE; if((dir->lockpush || cmd_getoption(a, "lock")) && !cmd_getoption(a, "nolock")) { if(dav_lock_t(root, dir->lock_timeout)) { print_resource_error(sn, "/"); dav_session_destroy(sn); fprintf(stderr, "Abort\n"); return -1; } DavLock *lock = dav_get_lock(sn, "/"); if(lock) { printf("Lock-Token: %s\n", lock->token); } locked = TRUE; locktokenfile = create_locktoken_file(dir->name, lock->token); } int sync_success = 0; int sync_error = 0; UcxList *resources = ucx_list_concat(modified, deleted); resources = ucx_list_sort(resources, (cmp_func)localres_cmp_path, NULL); UCX_FOREACH(elm, resources) { LocalResource *resource = elm->data; DavResource *res = dav_get(sn, resource->path, "D:getetag,idav:status,idav:version-collection,idav:split,`idav:content-hash`,idavprops:tags,idavprops:finfo,idavprops:xattributes,idavprops:link"); if(!res) { printf("skip: %s\n", resource->path); continue; } char *status = dav_get_string_property(res, "idav:status"); if(status && !strcmp(status, "broken")) { fprintf(stderr, "Resource %s broken\n", res->path); continue; } DavResource *vres = NULL; if(version) { if(dir->versioning->type == VERSIONING_SIMPLE) { vres = versioning_simple_find(res, version); } else if(dir->versioning->type == VERSIONING_DELTAV) { vres = versioning_deltav_find(res, version); } if(!vres) { fprintf(stderr, "Cannot find specified version for resource %s\n", res->path); ret = 1; break; } } else { vres = res; } // download the resource if(!sync_shutdown) { if(resource->isdirectory) { char *local_path = create_local_path(dir, res->path); if(sys_mkdir(local_path) && errno != EEXIST) { fprintf(stderr, "Cannot create directory %s: %s", local_path, strerror(errno)); } free(local_path); } else { if(sync_get_resource(a, dir, res->path, vres, db, &sync_success)) { fprintf(stderr, "sync_get_resource failed for resource: %s\n", res->path); sync_error++; } } } } // unlock repository if(locked) { if(dav_unlock(root)) { print_resource_error(sn, "/"); ret = -1; } else { locked = FALSE; } } // store db if(store_db(db, dir->database, dir->db_settings)) { fprintf(stderr, "Cannot store sync db\n"); ret = -2; } // cleanup dav_session_destroy(sn); if(!locked && locktokenfile) { remove(locktokenfile); } // Report if(ret != -2) { char *str_success = sync_success == 1 ? "file" : "files"; char *str_error = sync_error == 1 ? "error" : "errors"; printf("Result: %d %s pulled, %d %s\n", sync_success, str_success, sync_error, str_error); } return ret; } UcxList* local_scan(SyncDirectory *dir, SyncDatabase *db) { UcxList *resources = NULL; char *path = strdup("/"); UcxList *stack = ucx_list_prepend(NULL, path); while(stack) { // get a directory path from the stack and read all entries // if an entry is a directory, put it on the stack char *p = stack->data; stack = ucx_list_remove(stack, stack); char *local_path = create_local_path(dir, p); SYS_DIR local_dir = sys_opendir(local_path); if(!local_dir) { fprintf(stderr, "Cannot open directory %s\n", local_path); } else { SysDirEnt *ent; while((ent = sys_readdir(local_dir)) != NULL) { if(!strcmp(ent->name, ".") || !strcmp(ent->name, "..")) { continue; } char *new_path = util_concat_path(p, ent->name); DavBool free_new_path = TRUE; LocalResource *res = local_resource_new(dir, db, new_path); if(res) { if(res->isdirectory) { resources = ucx_list_append(resources, res); stack = ucx_list_prepend(stack, new_path); free_new_path = FALSE; } else { resources = ucx_list_append(resources, res); } } if(free_new_path) { free(new_path); } } sys_closedir(local_dir); } free(local_path); free(p); } return resources; } UcxList* read_changes(SyncDirectory *dir, SyncDatabase *db) { UcxProperties *parser = ucx_properties_new(); parser->delimiter = ':'; UcxList *resources = NULL; sstr_t name; sstr_t value; char buf[STDIN_BUF_SIZE]; size_t r; while(!feof(stdin)) { r = fread(buf, 1, STDIN_BUF_SIZE, stdin); ucx_properties_fill(parser, buf, r); while(ucx_properties_next(parser, &name, &value)) { if(value.length == 0) { fprintf(stderr, "Wrong input\n"); continue; } if(value.ptr[0] == '"' && value.length > 2 && value.ptr[value.length - 1] == '"') { value.ptr[value.length - 1] = '\0'; value.ptr++; value.length -= 2; } value = sstrdup(value); if(!sstrcmp(name, S("put"))) { LocalResource *res = local_resource_new(dir, db, value.ptr); if(res) { resources = ucx_list_append(resources, res); } } else if(!sstrcmp(name, S("remove"))) { ucx_map_sstr_remove(db->resources, value); } free(value.ptr); } } ucx_properties_free(parser); return resources; } LocalResource* local_resource_new(SyncDirectory *dir, SyncDatabase *db, char *path) { char *file_path = create_local_path(dir, path); SYS_STAT s; if(sys_lstat(file_path, &s)) { fprintf(stderr, "Cannot stat file %s\n", file_path); free(file_path); return NULL; } LocalResource *res = calloc(1, sizeof(LocalResource)); res->mode = s.st_mode & 07777; res->uid = s.st_uid; res->gid = s.st_gid; res->last_modified = s.st_mtime; if(!S_ISDIR(s.st_mode)) { res->path = strdup(path); res->size = s.st_size; } else { res->path = util_concat_path(path, "/"); res->isdirectory = 1; } int skip_file = 0; if(SYS_ISLINK(file_path, s)) { char *lnkbuf = sys_readlink(file_path, &s); #ifdef SYS_LINK_EXT // on Windows, we interpret .lnk files as links // we dont want resource names with this extension // and possibly sync this to other operating systems // therefore we remove the .lnk extension from the file name // change res->path // we only do this, if there isn't any other file with this name sstr_t fpath = sstr(file_path); sstr_t rpath = sstr(path); // remove last 4 chars (.lnk) sstr_t new_file_path = sstrdup(sstrsubsl(fpath, 0 , fpath.length-4)); // check if a file with this name exists SYS_STAT nfp_s; if(!sys_stat(new_file_path.ptr, &nfp_s)) { // we can't upload the link without the file extension, because // there is another file with this name free(lnkbuf); lnkbuf = NULL; } else { sstr_t new_path = sstrdup(sstrsubsl(rpath, 0, rpath.length-4)); res->local_path = res->path; res->path = new_path.ptr; // remove .lnk ext from resource path } free(new_file_path.ptr); #endif if(lnkbuf) { // readlink successful char *normalized = NULL; if(!util_path_isabsolut(lnkbuf)) { char *link_parent = util_parent_path(res->path); char *abs_link_parent = util_concat_path(dir->path, link_parent); char *link = util_concat_path(abs_link_parent, lnkbuf); normalized = util_path_normalize(link); free(abs_link_parent); free(link_parent); free(link); } else { normalized = util_path_normalize(lnkbuf); } char *dirpath = util_path_normalize(dir->path); int isintern = util_path_isrelated(dirpath, normalized); if(SYNC_SYMLINK(dir) && isintern) { // the link points to a file inside the syncdir char *rel = util_create_relative_path(normalized, file_path); res->link_target = rel; } else { #ifndef SYS_LINK_EXT // if not windows SYS_STAT targetstat; if(!sys_stat(file_path, &targetstat)) { res->isdirectory = S_ISDIR(targetstat.st_mode); int nofollowextern = (dir->symlink & SYNC_SYMLINK_IGNORE_EXTERN) == SYNC_SYMLINK_IGNORE_EXTERN; int nofollowintern = (dir->symlink & SYNC_SYMLINK_IGNORE_INTERN) == SYNC_SYMLINK_IGNORE_INTERN; if(isintern && nofollowintern) { skip_file = TRUE; } else if(!isintern && nofollowextern) { skip_file = TRUE; } } else { // can't access target, therefore we skip this link skip_file = TRUE; } #endif } free(dirpath); free(normalized); free(lnkbuf); } } if(!res->isdirectory && dir->push_strategy == PUSH_STRATEGY_HASH) { res->hash = util_file_hash(file_path); } free(file_path); if(skip_file) { local_resource_free(res); res = NULL; } return res; } char* local_resource_path(LocalResource *res) { return res->local_path ? res->local_path : res->path; } int local_resource_is_changed( SyncDirectory *dir, SyncDatabase *db, LocalResource *res, UcxMap *svrres, DavBool restore_removed, DavBool restore_modified) { LocalResource *db_res = ucx_map_cstr_get(db->resources, res->path); res->tags_updated = 0; if(db_res) { if(svrres) { DavResource *remote = ucx_map_cstr_get(svrres, res->path); if(restore_removed && !remote) { return 1; } if(!res->isdirectory && restore_modified && remote) { char *etag = dav_get_string_property(remote, "D:getetag"); if(!etag || (db_res->etag && strcmp(etag, db_res->etag))) { res->restore = TRUE; return 1; } } } // copy some metadata from db_res, that localscan does not deliver res->tags_updated = db_res->tags_updated; if(db_res->etag) { res->etag = strdup(db_res->etag); } if(db_res->tags_hash) { res->tags_hash = strdup(db_res->tags_hash); } if(db_res->remote_tags_hash) { res->remote_tags_hash = strdup(db_res->remote_tags_hash); } if(db_res->xattr_hash) { res->xattr_hash = strdup(db_res->xattr_hash); } if(db_res->hash) { res->prev_hash = strdup(db_res->hash); } // if the resource is splitted, move the part infos to the new // LocalResource obj, because we need it later if(db_res->parts) { res->parts = db_res->parts; res->numparts = db_res->numparts; db_res->parts = NULL; db_res->numparts = 0; } // check if metadata has changed // metadata are tags, mode, owner, xattr // set res->metadata_updated to 1 in case any metadata has changed // check if tags have changed if(dir->tagconfig && dir->tagconfig->detect_changes && !res->tags_updated) { UcxBuffer *tags = sync_get_file_tag_data(dir, res); if(tags) { if(db_res->tags_hash) { char *hash = dav_create_hash(tags->space, tags->size); if(strcmp(hash, db_res->tags_hash)) { res->tags_updated = 1; } free(hash); } else { res->tags_updated = 1; } } else if(db_res->tags_hash) { res->tags_updated = 1; // tags removed } res->metadata_updated = res->tags_updated; } // check if mode has changed if((dir->metadata & FINFO_MODE) == FINFO_MODE) { if(db_res->mode != res->mode) { res->finfo_updated = 1; res->metadata_updated = 1; } } // check if owner has changed if((dir->metadata & FINFO_OWNER) == FINFO_OWNER) { if(db_res->uid != res->uid || db_res->gid != res->gid) { res->finfo_updated = 1; res->metadata_updated = 1; } } // check if xattr have changed if((dir->metadata & FINFO_XATTR) == FINFO_XATTR) { char *path = create_local_path(dir, local_resource_path(db_res)); XAttributes *xattr = file_get_attributes(path, (xattr_filter_func)xattr_filter, dir); // test if xattr are added, removed or changed if((db_res->xattr_hash && !xattr) || (!db_res->xattr_hash && xattr) || (xattr && db_res->xattr_hash && strcmp(xattr->hash, db_res->xattr_hash))) { res->metadata_updated = 1; res->xattr_updated = 1; res->xattr = xattr; } else if(xattr) { xattributes_free(xattr); } } // check if the content of the file was modified // in case of links, just check if the link target has changed // for normal files, check last modified and size // or compare content hashes if(nullstrcmp(db_res->link_target, res->link_target)) { res->link_updated = 1; } else { if(db_res->hash && res->hash) { // we already have hashes if(!strcmp(db_res->hash, res->hash)) { return 0; // hashes equal -> file content unchanged } } else if( db_res->last_modified == res->last_modified && db_res->size == res->size && db_res->isdirectory == res->isdirectory) { // mtime and size unchanged, content also likely unchanged return 0; } else if(SYNC_HASHING(dir)) { // res->hash missing (see above) char *local_path = util_concat_path(dir->path, local_resource_path(res)); res->hash = util_file_hash(local_path); free(local_path); // check if the content has really changed if(res->hash && db_res->hash && !strcmp(res->hash, db_res->hash)) { return 0; } } } } else { res->tags_updated = 1; res->finfo_updated = 1; res->xattr_updated = 1; res->metadata_updated = 1; res->isnew = 1; } return 1; } int remote_resource_is_changed( DavSession *sn, SyncDirectory *dir, SyncDatabase *db, DavResource *remote, LocalResource *res, DavBool *equal) { if(equal) { *equal = FALSE; } DavPropName properties[] = { {"DAV:", "getetag"}, {DAV_NS, "version-collection"}, {DAV_NS, "content-hash"}, {DAV_NS, "split" }, {DAV_PROPS_NS, "tags"}, {DAV_PROPS_NS, "link" } }; int err = dav_load_prop(remote, properties, 6); if(res->restore) { return 0; } if(remote->iscollection) { return 1; } int ret = 0; if(err == 0) { char *etag = dav_get_string_property(remote, "D:getetag"); char *hash = sync_get_content_hash(remote); if(hash && res->hash && equal) { // if requested, check if the local and remote res are equal if(!strcmp(hash, res->hash)) { *equal = TRUE; return 0; } } if(hash && res->prev_hash) { if(strcmp(hash, res->prev_hash)) { ret = 1; } } else if(!res->etag) { // the resource is on the server and the client has no etag ret = 1; } else if(etag) { sstr_t e = sstr(etag); if(sstrprefix(e, S("W/"))) { e = sstrsubs(e, 2); } if(strcmp(e.ptr, res->etag)) { ret = 1; } } else { // something weird is happening, the server must support etags fprintf(stderr, "Warning: resource %s has no etag\n", remote->href); } } return ret; } int local_resource_load_metadata(SyncDirectory *dir, LocalResource *res) { // currently only xattr needed if((dir->metadata & FINFO_XATTR) == FINFO_XATTR) { char *path = create_local_path(dir, local_resource_path(res)); XAttributes *xattr = file_get_attributes(path, (xattr_filter_func)xattr_filter, dir); res->xattr = xattr; free(path); } return 0; } void local_resource_set_etag(LocalResource *local, const char *etag) { // free old etag if(local->etag) { free(local->etag); } if(!etag) { local->etag = NULL; return; } scstr_t e = scstr(etag); if(sstrprefix(e, S("W/"))) { e = scstrsubs(e, 2); } local->etag = sstrdup(e).ptr; } char* resource_local_path(DavResource *res) { #ifdef SYS_LINK_EXT // on Windows, add .lnk extension to links if(dav_get_property_ns(res, DAV_PROPS_NS, "link")) { return ucx_sprintf("%s%s", res->path, SYS_LINK_EXT).ptr; } else { // not a link return strdup(res->path); } #else return strdup(res->path); #endif } size_t resource_get_blocksize(SyncDirectory *dir, LocalResource *local, DavResource *res, off_t filesize) { size_t local_blocksize = 0; if(local->blocksize < 0) { // file splitting disabled return 0; } else if(local->blocksize > 0) { local_blocksize = (size_t)local->blocksize; } else { UCX_FOREACH(elm, dir->splitconfig) { SplitConfig *sc = elm->data; if(sc->filter) { if(res_matches_filter(sc->filter, local->path)) { continue; } } if(sc->minsize > 0) { if(filesize < sc->minsize) { continue; } } local_blocksize = sc->blocksize; break; } } size_t svr_blocksize = 0; char *svr_blocksize_str = dav_get_string_property_ns(res, DAV_NS, "split"); if(svr_blocksize_str) { uint64_t i = 0; if(util_strtouint(svr_blocksize_str, &i)) { svr_blocksize = (size_t)i; } } if(local_blocksize > 0 && svr_blocksize > 0) { fprintf(stderr, "Warning: Blocksize mismatch: %s: local: %zu server: %zu\n", local->path, local_blocksize, svr_blocksize); return svr_blocksize; } else if(local_blocksize > 0) { return local_blocksize; } else if(svr_blocksize > 0) { return svr_blocksize; } return 0; } int resource_pathlen_cmp(LocalResource *res1, LocalResource *res2, void *n) { size_t s1 = strlen(res1->path); size_t s2 = strlen(res2->path); if(s1 < s2) { return 1; } else if(s1 > s2) { return -1; } else { return 0; } } DavResource *versioning_simple_find(DavResource *res, const char *version) { char *vcol_href = dav_get_string_property_ns(res, DAV_NS, VERSION_PATH_PROPERTY); if(!vcol_href) { return NULL; } DavResource *vcol = dav_resource_new_href(res->session, vcol_href); if(!vcol) { return NULL; } if(dav_load_prop(vcol, defprops, numdefprops)) { print_resource_error(res->session, vcol->path); dav_resource_free(vcol); return NULL; } DavResource *ret = NULL; DavResource *child = vcol->children; while(child) { DavResource *next = child->next; if(!strcmp(child->name, version)) { ret = child; } else { dav_resource_free(child); } child = next; } dav_resource_free(vcol); return ret; } // TODO: remove code dup (main.c: find_version) DavResource* versioning_deltav_find(DavResource *res, const char *version) { DavResource *list = dav_versiontree(res, "D:getetag,idav:status,idav:split,idavprops:link,idavprops:finfo,idavprops:xattributes,idavprops:tags"); DavResource *ret = NULL; while(list) { DavResource *next = list->next; if(!ret) { char *vname = dav_get_string_property(list, "D:version-name"); if(vname && !strcmp(vname, version)) { ret = list; } } if(list != ret) { dav_resource_free(list); } list = next; } return ret; } int sync_set_status(DavResource *res, char *status) { DavResource *resource = dav_resource_new(res->session, res->path); dav_set_string_property(resource, "idav:status", status); int ret = dav_store(resource); dav_resource_free(resource); return ret; } int sync_remove_status(DavResource *res) { DavResource *resource = dav_resource_new(res->session, res->path); dav_remove_property(resource, "idav:status"); int ret = dav_store(resource); dav_resource_free(resource); return ret; } int sync_tags_equal(UcxList *tags1, UcxList *tags2) { if(!tags1) { return tags2 ? 0 : 1; } if(!tags2) { return tags1 ? 0 : 1; } UcxMap *map1 = ucx_map_new(32); UCX_FOREACH(elm, tags1) { DavTag *t = elm->data; ucx_map_cstr_put(map1, t->name, t); } int equal = 1; int i = 0; UCX_FOREACH(elm, tags2) { DavTag *t = elm->data; if(!ucx_map_cstr_get(map1, t->name)) { equal = 0; break; } i++; } if(i != map1->count) { equal = 0; } ucx_map_free(map1); return equal; } int sync_store_metadata(SyncDirectory *dir, const char *path, LocalResource *local, DavResource *res) { int ret = 0; DavXmlNode *fileinfo = dav_get_property_ns(res, DAV_PROPS_NS, "finfo"); if(fileinfo) { FileInfo f; finfo_get_values(fileinfo, &f); if((dir->metadata & FINFO_MTIME) == FINFO_MTIME && f.date_set) { // set mtime struct utimbuf t; t.actime = f.last_modified; t.modtime = f.last_modified; if(utime(path, &t)) { fprintf(stderr, "utime failed for file: %s : %s\n", path, strerror(errno)); ret = 1; } } if((dir->metadata & FINFO_MODE) == FINFO_MODE && f.mode_set) { // set mode if(chmod(path, f.mode)) { fprintf(stderr, "chmod failed for file: %s : %s\n", path, strerror(errno)); ret = 1; } } } DavXmlNode *xattr_prop = dav_get_property_ns(res, DAV_PROPS_NS, "xattributes"); if(xattr_prop) { XAttributes *xattr = xml_get_attributes(xattr_prop); if(xattr) { if(!sync_store_xattr(dir, path, xattr)) { if(local->xattr_hash) { free(local->xattr_hash); } local->xattr_hash = xattr->hash; } } } if(sync_store_tags(dir, path, local, res)) { ret = 1; } return ret; } int sync_store_xattr(SyncDirectory *dir, const char *path, XAttributes *xattr) { for(int i=0;i<xattr->nattr;i++) { sstr_t value = xattr->values[i]; if(xattr_set(path, xattr->names[i], value.ptr, value.length)) { fprintf( stderr, "Cannot store xattr '%s' for file: %s\n", xattr->names[i], path); } } return 0; } int sync_store_tags(SyncDirectory *dir, const char *path, LocalResource *local, DavResource *res) { if(!dir->tagconfig) { return 0; } UcxList *tags = NULL; if(dir->tagconfig) { DavXmlNode *tagsprop = dav_get_property_ns(res, DAV_PROPS_NS, "tags"); if(tagsprop) { tags = parse_dav_xml_taglist(tagsprop); } } DavBool store_tags = FALSE; DavBool tags_changed = FALSE; UcxList *local_tags = sync_get_file_tags(dir, local, &tags_changed, NULL); if(tags_changed) { switch(dir->tagconfig->conflict) { case TAG_NO_CONFLICT: { store_tags = TRUE; break; } case TAG_KEEP_LOCAL: { store_tags = FALSE; break; } case TAG_KEEP_REMOTE: { store_tags = TRUE; local->tags_updated = FALSE; break; } case TAG_MERGE: { UcxList *new_tags = merge_tags(local_tags, tags); // TODO: free tags and local_tags tags = new_tags; store_tags = TRUE; // make sure the merged tags will be pushed the next time local->tags_updated = TRUE; break; } } } else { if(!sync_tags_equal(tags, local_tags)) { store_tags = TRUE; } // TODO: free local_tags } if(!store_tags) { return 0; } int ret = sync_store_tags_local(dir, local, path, tags); // TODO: free stuff return ret; } int sync_store_tags_local(SyncDirectory *dir, LocalResource *local, const char *path, UcxList *tags) { int ret = 0; if(dir->tagconfig->store == TAG_STORE_XATTR) { UcxBuffer *data = NULL; if(tags) { switch(dir->tagconfig->local_format) { default: break; case TAG_FORMAT_TEXT: { data = create_text_taglist(tags); break; } case TAG_FORMAT_CSV: { data = create_csv_taglist(tags); break; } case TAG_FORMAT_MACOS: { data = create_macos_taglist(tags); break; } } if(data) { char *data_hash = dav_create_hash(data->space, data->size); int update = 1; if(local) { if(!local->tags_hash || strcmp(data_hash, local->tags_hash)) { //printf("update: %s\n", local->path); } else { update = 0; } } if(update) { ret = xattr_set(path, dir->tagconfig->xattr_name, data->space, data->pos); if(local) { if(local->tags_hash) { free(local->tags_hash); local->tags_hash = NULL; } local->tags_hash = data_hash; } } else { free(data_hash); } ucx_buffer_free(data); } else { ret = -1; } } else { if(local) { //printf("update: %s\n", local->path); } // ignore errors on remove xattr_remove(path, dir->tagconfig->xattr_name); } } if(!ret && local) { local->tags_updated = 0; } return ret; } UcxBuffer* sync_get_file_tag_data(SyncDirectory *dir, LocalResource *res) { if(!dir->tagconfig) { return NULL; } if(res->cached_tags) { return res->cached_tags; } UcxBuffer *buf = NULL; if(dir->tagconfig->store == TAG_STORE_XATTR) { ssize_t tag_length = 0; char *local_path = create_local_path(dir, local_resource_path(res)); char* tag_data = xattr_get( local_path, dir->tagconfig->xattr_name, &tag_length); free(local_path); if(tag_length > 0) { buf = ucx_buffer_new(tag_data, (size_t)tag_length, UCX_BUFFER_AUTOFREE); buf->size = (size_t)tag_length; } } res->cached_tags = buf; return buf; } UcxList* sync_get_file_tags(SyncDirectory *dir, LocalResource *res, DavBool *changed, char **newhash) { if(changed) *changed = FALSE; UcxList *tags = NULL; if(!res) { return NULL; } if(!dir->tagconfig) { return NULL; } if(changed && res->tags_updated) { *changed = TRUE; } if(dir->tagconfig->store == TAG_STORE_XATTR) { UcxBuffer *tag_buf = res->cached_tags ? res->cached_tags : sync_get_file_tag_data(dir, res); if(tag_buf) { char *new_hash = dav_create_hash(tag_buf->space, tag_buf->size); if(res->tags_hash) { if(changed && strcmp(res->tags_hash, new_hash)) { *changed = TRUE; } free(res->tags_hash); res->tags_hash = NULL; } else { if(changed) *changed = TRUE; } if(newhash) { *newhash = new_hash; } else { free(new_hash); } switch(dir->tagconfig->local_format) { default: break; case TAG_FORMAT_TEXT: { tags = parse_text_taglist(tag_buf->space, tag_buf->size); break; } case TAG_FORMAT_CSV: { tags = parse_csv_taglist(tag_buf->space, tag_buf->size); break; } case TAG_FORMAT_MACOS: { tags = parse_macos_taglist(tag_buf->space, tag_buf->size); break; } } res->cached_tags = tag_buf; } else if(res->tags_hash) { if(changed) *changed = TRUE; } } return tags; } static int file_seek(FILE *f, curl_off_t offset, int origin) { int ret = fseek(f, offset, origin); return ret == 0 ? CURL_SEEKFUNC_OK : CURL_SEEKFUNC_CANTSEEK; } size_t myread(void *ptr, size_t size, size_t nmemb, FILE *f) { size_t ret = fread(ptr, size, nmemb, f); return ret; } int gen_random_name(char *buf, size_t len) { unsigned char name_prefix[8]; memset(name_prefix, 0, 8); dav_rand_bytes(name_prefix, 8); char *pre = util_hexstr(name_prefix, 8); int64_t ts = (int64_t)time(NULL); int w = snprintf(buf, len, "%"PRId64"-%s", ts, pre); free(pre); return w >= len; } #define VBEGIN_ERROR_MKCOL 1 #define VBEGIN_ERROR_MOVE 2 #define VBEGIN_ERROR_PROPPATCH 3 #define VBEGIN_ERROR_CHECKOUT 4 int versioning_begin(SyncDirectory *dir, DavResource *res, int *exists) { int ret = 0; if(dir->versioning->type == VERSIONING_SIMPLE && res->exists) { DavResource *history_collection = dav_resource_new( res->session, dir->versioning->collection); // get the path to the version history collection for this resource // if propfind fails we just assume that it doesn't exist // better error handling is done later (sync_put_resource) // if there is no history collection for this resource, we create one char *history_href = NULL; char *vcol_path = dav_get_string_property_ns(res, DAV_NS, VERSION_PATH_PROPERTY); if(!vcol_path) { DavResource *history_res = NULL; // create a new collection for version history // the name is a combination of a random prefix and a timestamp while(!history_res) { char history_res_name[128]; gen_random_name(history_res_name, 128); history_res = dav_resource_new_child( res->session, history_collection, history_res_name); if(dav_exists(history_res)) { dav_resource_free(history_res); history_res = NULL; } } history_res->iscollection = TRUE; if(dav_create(history_res)) { dav_resource_free(history_res); dav_resource_free(history_collection); return VBEGIN_ERROR_MKCOL; } history_href = strdup(history_res->href); dav_resource_free(history_res); } else { history_href = vcol_path; } // find a free url and move 'res' to this location DavResource *version_res = NULL; while(!version_res) { char version_name[128]; gen_random_name(version_name, 128); char *href = util_concat_path(history_href, version_name); version_res = dav_resource_new_href(res->session, href); free(href); char *dest = util_get_url(res->session, version_res->href); int err = dav_moveto(res, dest, FALSE); free(dest); if(err) { dav_resource_free(version_res); version_res = NULL; if(res->session->error != DAV_PRECONDITION_FAILED) { ret = VBEGIN_ERROR_MOVE; break; } } else { // tell the caller the resource does not exist anymore at // the original location *exists = 0; } } if(!ret) { dav_set_string_property_ns(version_res, DAV_NS, "origin", res->href); if(dav_store(version_res)) { ret = VBEGIN_ERROR_PROPPATCH; } dav_resource_free(version_res); // we can just set the property here and don't need dav_store // because sync_put_resource will call dav_store(res) later dav_set_string_property_ns( res, DAV_NS, VERSION_PATH_PROPERTY, history_href); } if(vcol_path != history_href) { free(history_href); } dav_resource_free(history_collection); } else if(dir->versioning->type == VERSIONING_DELTAV){ // DeltaV is so much easier :) if(dav_checkout(res)) { ret = VBEGIN_ERROR_CHECKOUT; } } return ret; } int versioning_end(SyncDirectory *dir, DavResource *res) { if(dir->versioning->type == VERSIONING_DELTAV) { return dav_checkin(res); } else { return 0; } } int versioning_delete(SyncDirectory *dir, DavResource *res) { if(dir->versioning->type == VERSIONING_SIMPLE) { // TODO } return 0; } static void update_metadata_hashes(LocalResource *local, MetadataHashes hashes) { if(hashes.tags) { if(local->tags_hash) { free(local->tags_hash); local->tags_hash = NULL; } local->tags_hash = hashes.tags; } if(hashes.tags_remote) { if(local->remote_tags_hash) { free(local->remote_tags_hash); } local->remote_tags_hash = hashes.tags_remote; } if(hashes.xattr) { if(local->xattr_hash) { free(local->xattr_hash); } local->xattr_hash = hashes.xattr; } } // this macro is only a workaround for a netbeans bug #define LOG10 log10 static UcxList* upload_parts( LocalResource *local, DavResource *res, FILE *in, uint64_t filesize, size_t blocksize, uint64_t *blockcount, int *err) { // Make sure the resource is a collection. If it was a normal // resource until now, delete it and recreate it as collection if(res->exists) { if(!res->iscollection) { if(dav_delete(res)) { print_resource_error(res->session, res->path); *err = 1; return NULL; } res->exists = 0; return upload_parts(local, res, in, filesize, blocksize, blockcount, err); } } else { res->iscollection = 1; if(dav_create(res)) { print_resource_error(res->session, res->path); *err = 1; return NULL; } } res->exists = 1; if(!res->href) { // this should never happen, but just make sure it doesn't crash fprintf(stderr, "href is NULL\n"); *err = 1; return NULL; } char *buffer = malloc(blocksize); if(!buffer) { fprintf(stderr, "Out of memory\n"); *err = 1; return NULL; } // calculate the maximal length of resource names // names should have all the same length and contain the block number int nblocks = filesize / blocksize; int digits = LOG10((double)nblocks) + 1; if(digits > 127) { fprintf(stderr, "Too many parts\n"); *err = 1; return NULL; } UcxMap *updated_parts_map = ucx_map_new((nblocks/2)+64); int blockindex = 0; int uploaded_parts = 0; size_t r; // temporarly disable name encryption, because we don't need it for // part names uint32_t session_flags = res->session->flags; res->session->flags ^= DAV_SESSION_ENCRYPT_NAME; DAV_SHA_CTX *sha = dav_hash_init(); while((r = fread(buffer, 1, blocksize, in)) > 0) { dav_hash_update(sha, buffer, r); int upload_block = 0; char *block_hash = dav_create_hash(buffer, r); if(blockindex >= local->numparts) { // we don't have a hash for this block, therefore it must be new upload_block = 1; } else { FilePart part = local->parts[blockindex]; if(!strcmp(part.hash, block_hash)) { // no change free(block_hash); block_hash = NULL; } else { // block has changed upload_block = 1; } } if(upload_block) { char name[128]; snprintf(name, 128, "%0*d", digits, blockindex); char *part_href = util_concat_path(res->href, name); DavResource *part = dav_resource_new_href(res->session, part_href); free(part_href); // upload part dav_set_content_data(part, buffer, r); if(dav_store(part)) { *err = 1; print_resource_error(res->session, part->path); } else { // successfully uploaded part // store the FilePart in a map // later we do a propfind and add the etag FilePart *f = calloc(1, sizeof(FilePart)); f->block = blockindex; f->hash = block_hash; ucx_map_cstr_put(updated_parts_map, name, f); } dav_resource_free(part); uploaded_parts++; } if(*err) { break; } blockindex++; } *blockcount = blockindex; // restore flags res->session->flags = session_flags; free(buffer); if(*err) { ucx_map_free_content(updated_parts_map, (ucx_destructor)filepart_free); ucx_map_free(updated_parts_map); return NULL; } // set content-hash unsigned char content_hash[DAV_SHA256_DIGEST_LENGTH]; dav_hash_final(sha, content_hash); sync_set_content_hash(res, content_hash); local->hash = util_hexstr(content_hash, DAV_SHA256_DIGEST_LENGTH); // get etags from uploaded resources // also delete everything, that is not part of the file UcxList *updated_parts = NULL; DavResource *parts = dav_query(res->session, "select D:getetag from %s order by name", res->path); if(!parts) { print_resource_error(res->session, parts->path); *err = 1; ucx_map_free_content(updated_parts_map, (ucx_destructor)filepart_free); ucx_map_free(updated_parts_map); return NULL; } DavResource *part = parts->children; while(part) { FilePart *fp = ucx_map_cstr_remove(updated_parts_map, part->name); // every part we uploaded is in the map // if we get parts that are not in the map, someone else uploaded it if(fp) { char *etag = dav_get_string_property(part, "D:getetag"); if(etag) { if(strlen(etag) > 2 && etag[0] == 'W' && etag[1] == '/') { etag = etag + 2; } fp->etag = strdup(etag); updated_parts = ucx_list_append(updated_parts, fp); } // else { wtf is wrong with this resource } } else { uint64_t name_partnum = 0; char *res_name = part->name; while(res_name[0] == '0' && res_name[1] != '\0') { res_name++; } DavBool delete_part = 0; if(strlen(part->name) != digits) { delete_part = 1; } else if(util_strtouint(res_name, &name_partnum)) { if(name_partnum >= blockindex) { delete_part = 1; } } if(delete_part) { if(dav_delete(part)) { print_resource_error(part->session, part->path); } } } part = part->next; } dav_resource_free_all(parts); ucx_map_free_content(updated_parts_map, (ucx_destructor)filepart_free); ucx_map_free(updated_parts_map); *err = 0; return updated_parts; } void update_parts(LocalResource *local, UcxList *updates, uint64_t numparts) { size_t old_num = local->numparts; if(old_num > numparts) { // free old parts for(size_t i=numparts;i<old_num;i++) { FilePart p = local->parts[i]; if(p.etag) { free(p.etag); } if(p.hash) { free(p.hash); } } } if(numparts != local->numparts) { local->parts = realloc(local->parts, numparts * sizeof(FilePart)); local->numparts = numparts; } UCX_FOREACH(elm, updates) { FilePart *p = elm->data; if(p->block > numparts) { // just make sure things don't explode in case some weird stuff // is going on continue; } FilePart *old = &local->parts[p->block]; if(p->block < old_num) { // cleanup existing part if(old->hash) { free(old->hash); old->hash = NULL; } if(old->etag) { free(old->etag); old->etag = NULL; } } old->block = p->block; old->hash = p->hash; old->etag = p->etag; free(p); } } int sync_put_resource( SyncDirectory *dir, DavResource *res, LocalResource *local, int *counter) { char *local_path = create_local_path(dir, local_resource_path(local)); SYS_STAT s; if(sys_stat(local_path, &s)) { fprintf(stderr, "Cannot stat file: %s\n", local_path); perror(""); free(local_path); return -1; } DavBool islink = local->link_target ? 1 : 0; if(!local->link_target && local->link_updated) { dav_remove_property_ns(res, DAV_PROPS_NS, "link"); } size_t split_blocksize = resource_get_blocksize(dir, local, res, s.st_size); FILE *in = sys_fopen(local_path, "rb"); if(!in) { fprintf(stderr, "Cannot open file %s\n", local_path); free(local_path); return -1; } DavBool issplit = split_blocksize == 0 ? FALSE : TRUE; int split_err = 0; UcxList *parts = NULL; uint64_t blockcount = 0; if(islink) { dav_set_string_property_ns(res, DAV_PROPS_NS, "link", local->link_target); } else if(issplit) { // set split property char blocksize_str[32]; snprintf(blocksize_str, 32, "%zu", split_blocksize); dav_set_string_property_ns(res, DAV_NS, "split", blocksize_str); // splitted/partial upload parts = upload_parts( local, res, in, s.st_size, split_blocksize, &blockcount, &split_err); } else { // regular file upload dav_set_content(res, in, (dav_read_func)myread, (dav_seek_func)file_seek); dav_set_content_length(res, s.st_size); } if(split_err) { free(local_path); return -1; } MetadataHashes hashes; hashes = sync_set_metadata_properties(dir, res->session, res, local); // before sync_put_resource, remote_resource_is_changed does a propfind // and sets res->exists int exists = res->exists; if(dir->versioning && dir->versioning->always && !issplit) { int err = versioning_begin(dir, res, &exists); if(err) { fprintf(stderr, "Cannot store version for resource: %s\n", res->href); free(local_path); return -1; } } int ret = -1; for(int i=0;i<=dir->max_retry;i++) { if(!exists && dav_create(res)) { continue; } exists = 1; if(dav_store(res)) { continue; } ret = 0; break; } if(dir->versioning && dir->versioning->always && !issplit) { if(versioning_end(dir, res)) { fprintf(stderr, "Cannot checkin resource\n"); ret = 1; } } if(ret == 0) { (*counter)++; update_metadata_hashes(local, hashes); update_parts(local, parts, blockcount); // check contentlength and get new etag DavResource *up_res = dav_get(res->session, res->path, "D:getetag,idav:status"); if(up_res) { // the new content length must be equal or greater than the file size if(up_res->contentlength < s.st_size && !issplit && !islink) { fprintf(stderr, "Incomplete Upload: %s\n", local_path); ret = -1; // try to set the resource status to 'broken' sync_set_status(res, "broken"); } else { // everything seems fine, we can update the local resource char *etag = dav_get_string_property(up_res, "D:getetag"); local_resource_set_etag(local, etag); if(!issplit && SYNC_STORE_HASH(dir)) { if(local->hash) { free(local->hash); } // TODO: calculate hash on upload local->hash = util_file_hash(local_path); } if(dav_get_string_property(up_res, "idav:status")) { sync_remove_status(up_res); } dav_resource_free(up_res); } } } else { ret = -1; sync_set_status(res, "broken"); } fclose(in); free(local_path); return ret; } int sync_mkdir(SyncDirectory *dir, DavResource *res, LocalResource *local) { res->iscollection = 1; int ret = -1; for(int i=0;i<=dir->max_retry;i++) { if(dav_create(res)) { continue; } ret = 0; break; } return ret; } int sync_move_remote_resource( SyncDirectory *dir, SyncDatabase *db, DavResource *origin, LocalResource *local, DavBool copy, int *counter) { char *local_path = create_local_path(dir, local->path); SYS_STAT s; if(sys_stat(local_path, &s)) { fprintf(stderr, "Cannot stat file: %s\n", local_path); perror(""); free(local_path); return -1; } free(local_path); int result = 0; if(copy) { result = dav_copy_o(origin, local->path, FALSE); } else { result = dav_move_o(origin, local->path, FALSE); } if(result != 0) { return result; } LocalResource *local_origin = local->origin; if(!copy) { ucx_map_cstr_remove(db->resources, local_origin->path); } // replace LocalResource with origin content local->origin = NULL; char *path = strdup(local->path); // TODO: free stuff before replacing it memcpy(local, local_origin, sizeof(LocalResource)); local->path = path; free(local_origin); // only free origin pointer // get new etag DavResource *up_res = dav_get(origin->session, local->path, "D:getetag"); if(up_res) { (*counter)++; // set metadata MetadataHashes hashes; hashes = sync_set_metadata_properties(dir, up_res->session, up_res, local); if(dav_store(up_res)) { fprintf(stderr, "Error: cannot store resource metadata\n"); } // everything seems fine, we can update the local resource char *etag = dav_get_string_property(up_res, "D:getetag"); local_resource_set_etag(local, etag); local->last_modified = s.st_mtime; dav_resource_free(up_res); } else { result = 1; } return result; } int sync_delete_remote_resource( SyncDirectory *dir, DavSession *sn, LocalResource *local_res, int *counter, UcxList **cols) { DavResource *res = dav_get(sn, local_res->path, "D:getetag,idav:split"); if(!res) { return sn->error == DAV_NOT_FOUND ? 0 : 1; } int ret = 0; sn->error = DAV_OK; if(res->iscollection) { DavXmlNode *split = dav_get_property_ns(res, DAV_NS, "split"); if(cols) { *cols = ucx_list_append(*cols, local_res); } else if(split || !res->children) { printf("delete: %s\n", res->path); if(dav_delete(res)) { ret = 1; fprintf(stderr, "Cannot delete collection %s\n", res->path); } else { (*counter)++; } } } else { char *etag = dav_get_string_property(res, "D:getetag"); if(etag) { if(strlen(etag) > 2 && etag[0] == 'W' && etag[1] == '/') { etag = etag + 2; } } if(!nullstrcmp(etag, local_res->etag)) { // local resource metadata == remote resource metadata // resource can be deleted printf("delete: %s\n", res->path); if(dir->versioning && dir->versioning->always) { if(versioning_delete(dir, res)) { fprintf( stderr, "Cannot save resource version before deletion\n"); ret = 1; } } if(!ret && dav_delete(res)) { if(sn->error != DAV_NOT_FOUND) { fprintf(stderr, "Cannot delete resource %s\n", res->path); ret = 1; } } else { (*counter)++; } } // else TODO: should we inform the user that the file was modified on // the server and delete was skipped? } // cleanup dav_resource_free(res); return ret; } MetadataHashes sync_set_metadata_properties( SyncDirectory *dir, DavSession *sn, DavResource *res, LocalResource *local) { MetadataHashes hashes = {NULL, NULL, NULL}; if(dir->tagconfig) { // get local tags DavBool changed = 0; char *tags_hash = NULL; UcxList *tags = sync_get_file_tags(dir, local, &changed, &tags_hash); if(changed || local->tags_updated) { hashes.tags = tags_hash; DavBool store_tags = TRUE; // get remote tags UcxList *remote_tags = NULL; DavXmlNode *tagsprop = dav_get_property_ns(res, DAV_PROPS_NS, "tags"); if(tagsprop) { remote_tags = parse_dav_xml_taglist(tagsprop); } char *remote_hash = create_tags_hash(remote_tags); if(nullstrcmp(remote_hash, local->remote_tags_hash)) { // the tags have changed on the server switch(dir->tagconfig->conflict) { case TAG_NO_CONFLICT: break; case TAG_KEEP_LOCAL: break; case TAG_KEEP_REMOTE: { store_tags = FALSE; local->tags_updated = FALSE; break; } case TAG_MERGE: { UcxList *new_tags = merge_tags(tags, remote_tags); free_taglist(tags); tags = new_tags; break; } } } if(dir->tagconfig->local_format == TAG_FORMAT_CSV) { // csv tag lists don't have colors, so we have to add // the colors from the remote tag list add_tag_colors(tags, remote_tags); } if(store_tags) { if(tags) { DavXmlNode *tagprop = create_xml_taglist(tags); dav_set_property_ns(res, DAV_PROPS_NS, "tags", tagprop); } else { dav_remove_property_ns(res, DAV_PROPS_NS, "tags"); } } free_taglist(remote_tags); } else { if(tags_hash) { free(tags_hash); } } free_taglist(tags); } if(local->finfo_updated) { struct stat s; s.st_mode = local->mode; s.st_mtime = local->last_modified; s.st_uid = local->uid; s.st_gid = local->gid; resource_set_finfo_s(&s, res, dir->metadata); } if(local->xattr_updated) { if(local->xattr) { resource_set_xattr(res, local->xattr); hashes.xattr = strdup(local->xattr->hash); } else { dav_remove_property(res, "idavprops:xattributes"); } } return hashes; } int sync_update_metadata( SyncDirectory *dir, DavSession *sn, DavResource *res, LocalResource *local) { MetadataHashes hashes = sync_set_metadata_properties(dir, sn, res, local); int err = 0; if(dav_store(res)) { print_resource_error(sn, local->path); err = 1; } else { update_metadata_hashes(local, hashes); } return err; } void remove_deleted_conflicts(SyncDirectory *dir, SyncDatabase *db) { char **dc = calloc(sizeof(void*), db->conflict->count); int numdc = 0; UcxMapIterator i = ucx_map_iterator(db->conflict); LocalResource *res; UCX_MAP_FOREACH(key, res, i) { char *path = create_local_path(dir, res->path); SYS_STAT s; if(sys_stat(path, &s)) { if(errno == ENOENT) { dc[numdc] = res->path; numdc++; } else { fprintf(stderr, "Cannot stat file: %s\n", path); perror(""); } } free(path); } for(int i=0;i<numdc;i++) { ucx_map_cstr_remove(db->conflict, dc[i]); } free(dc); } static void resolve_skipped(SyncDatabase *db) { UcxKey k; LocalResource *res; UcxMapIterator i = ucx_map_iterator(db->resources); int skipped = 0; UCX_MAP_FOREACH(k, res, i) { if(res->skipped) { skipped++; fprintf(stderr, "skipped from push: %s\n", res->path); } } if(skipped > 0) { fprintf(stderr, " To resolve conflict resources skipped by push run dav-sync pull first\n" " before resolve-conflicts or delete-conflicts.\n\n"); } } int cmd_resolve_conflicts(CmdArgs *a) { if(a->argc != 1) { fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many"); return -1; } SyncDirectory *dir = scfg_get_dir(a->argv[0]); if(!dir) { fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]); return -1; } if(scfg_check_dir(dir)) { return -1; } SyncDatabase *db = load_db(dir->database); if(!db) { fprintf(stderr, "Cannot load database file: %s\n", dir->database); return -1; } resolve_skipped(db); int ret = 0; // remove conflicts int num_conflict = db->conflict->count; ucx_map_free_content(db->conflict, (ucx_destructor)local_resource_free); ucx_map_clear(db->conflict); // store db if(store_db(db, dir->database, dir->db_settings)) { fprintf(stderr, "Cannot store sync db\n"); fprintf(stderr, "Abort\n"); ret = -2; } // cleanup destroy_db(db); // Report if(ret != -2) { char *str_conflict = num_conflict == 1 ? "conflict" : "conflicts"; printf("Result: %d %s resolved\n", num_conflict, str_conflict); } return ret; } int cmd_delete_conflicts(CmdArgs *a) { if(a->argc != 1) { fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many"); return -1; } SyncDirectory *dir = scfg_get_dir(a->argv[0]); if(!dir) { fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]); return -1; } if(scfg_check_dir(dir)) { return -1; } SyncDatabase *db = load_db(dir->database); if(!db) { fprintf(stderr, "Cannot load database file: %s\n", dir->database); return -1; } resolve_skipped(db); int num_del = 0; int num_err = 0; int ret = 0; // delete all conflict files UcxMapIterator i = ucx_map_iterator(db->conflict); LocalResource *res; UCX_MAP_FOREACH(key, res, i) { printf("delete: %s\n", res->path); char *path = create_local_path(dir, res->path); if(sys_unlink(path)) { if(errno != ENOENT) { perror("unlink"); num_err++; } } else { num_del++; } free(path); } ucx_map_free_content(db->conflict, (ucx_destructor)local_resource_free); ucx_map_clear(db->conflict); // store db if(store_db(db, dir->database, dir->db_settings)) { fprintf(stderr, "Cannot store sync db\n"); fprintf(stderr, "Abort\n"); ret = -1; } // cleanup destroy_db(db); // Report if(ret == 0) { char *str_delete = num_del == 1 ? "file" : "files"; char *str_error = num_err == 1 ? "error" : "errors"; printf("Result: %d conflict %s deleted, %d %s\n", num_del, str_delete, num_err, str_error); } return ret; } int cmd_list_conflicts(CmdArgs *a) { if(a->argc != 1) { fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many"); return -1; } SyncDirectory *dir = scfg_get_dir(a->argv[0]); if(!dir) { fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]); return -1; } if(scfg_check_dir(dir)) { return -1; } SyncDatabase *db = load_db(dir->database); if(!db) { fprintf(stderr, "Cannot load database file: %s\n", dir->database); return -1; } remove_deleted_conflicts(dir, db); // get all conflict sources UcxMapIterator i = ucx_map_iterator(db->conflict); LocalResource *res; UcxList* conflict_sources = NULL; UCX_MAP_FOREACH(key, res, i) { conflict_sources = ucx_list_append(conflict_sources, res->conflict_source); } // print unique conflict sources conflict_sources = ucx_list_sort(conflict_sources, ucx_cmp_str, NULL); UCX_FOREACH(elem, conflict_sources) { char* path = elem->data; if(cmd_getoption(a, "verbose")) { int confl_count = 1; while(elem->next && !strcmp(elem->next->data, path)) { elem = elem->next; ++confl_count; } printf("%s (%d)\n", path, confl_count); } else { printf("%s\n", path); while(elem->next && !strcmp(elem->next->data, path)) { elem = elem->next; } } } // cleanup destroy_db(db); return 0; } // TODO: remove code dup (main.c ls_size_str) static char* size_str(uint64_t size) { char *str = malloc(16); if(size < 0x400) { snprintf(str, 16, "%" PRIu64 " bytes", size); } else if(size < 0x100000) { float s = (float)size/0x400; int diff = (s*100 - (int)s*100); if(diff > 90) { diff = 0; s += 0.10f; } if(size < 0x2800 && diff != 0) { // size < 10 KiB snprintf(str, 16, "%.1f KiB", s); } else { snprintf(str, 16, "%.0f KiB", s); } } else if(size < 0x40000000) { float s = (float)size/0x100000; int diff = (s*100 - (int)s*100); if(diff > 90) { diff = 0; s += 0.10f; } if(size < 0xa00000 && diff != 0) { // size < 10 MiB snprintf(str, 16, "%.1f MiB", s); } else { size /= 0x100000; snprintf(str, 16, "%.0f MiB", s); } } else if(size < 0x1000000000ULL) { float s = (float)size/0x40000000; int diff = (s*100 - (int)s*100); if(diff > 90) { diff = 0; s += 0.10f; } if(size < 0x280000000 && diff != 0) { // size < 10 GiB snprintf(str, 16, "%.1f GiB", s); } else { size /= 0x40000000; snprintf(str, 16, "%.0f GiB", s); } } else { size /= 1024; float s = (float)size/0x40000000; int diff = (s*100 - (int)s*100); if(diff > 90) { diff = 0; s += 0.10f; } if(size < 0x280000000 && diff != 0) { // size < 10 TiB snprintf(str, 16, "%.1f TiB", s); } else { size /= 0x40000000; snprintf(str, 16, "%.0f TiB", s); } } return str; } void print_resource_version(DavResource *res, char *name) { time_t now = res->lastmodified; struct tm *date = gmtime(&now); char str[32]; putenv("LC_TIME=C"); size_t len = strftime(str, 32, "%a, %d %b %Y %H:%M:%S GMT", date); printf("name: %s\n", name); printf("lastmodified: %s\n", str); char *server = util_url_base(res->session->base_url); char *url = util_concat_path(server, res->href); printf("url: %s\n", url); free(server); free(url); } int cmd_list_versions(CmdArgs *a) { if(a->argc != 1) { fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many"); return -1; } SyncFile file; int ret = 0; char *path = a->argv[0]; int err = sync_get_file(a, path, &file, TRUE); if(err) { sync_print_get_file_err(path, err); return 1; } SyncDirectory *dir = file.dir; if(!dir->versioning) { fprintf(stderr, "No versioning configured for syncdir %s\n", dir->name); } Repository *repo = get_repository(sstr(dir->repository)); if(!repo) { fprintf(stderr, "Unknown repository %s\n", dir->repository); return -1; } SyncDatabase *db = load_db(dir->database); if(!db) { fprintf(stderr, "Cannot load database file: %s\n", dir->database); return -1; } remove_deleted_conflicts(dir, db); DavSession *sn = create_session(ctx, repo, dir->collection); ucx_mempool_reg_destr(sn->mp, db, (ucx_destructor)destroy_db); if (cmd_getoption(a, "verbose")) { curl_easy_setopt(sn->handle, CURLOPT_VERBOSE, 1L); curl_easy_setopt(sn->handle, CURLOPT_STDERR, stderr); } DavResource *res = dav_resource_new(sn, file.path); if(dir->versioning->type == VERSIONING_SIMPLE) { do { DavPropName p; p.ns = DAV_NS; p.name = VERSION_PATH_PROPERTY; if(dav_load_prop(res, &p, 1)) { print_resource_error(sn, file.path); ret = 1; break; } char *vcol_href = dav_get_string_property_ns(res, DAV_NS, VERSION_PATH_PROPERTY); if(!vcol_href) { ret = 1; break; } DavResource *vcol = dav_resource_new_href(sn, vcol_href); if(!vcol) { ret = 1; break; } if(dav_load_prop(vcol, NULL, 0)) { print_resource_error(sn, vcol->path); ret = 1; break; } DavResource *child = vcol->children; UcxList *children = NULL; while(child) { children = ucx_list_append(children, child); child = child->next; } children = ucx_list_sort(children, ucx_cmp_str, NULL); DavBool first = 1; UCX_FOREACH(elm, children) { DavResource *c = elm->data; if(!first) { putchar('\n'); } print_resource_version(c, c->name); first = 0; } ucx_list_free(children); } while(0); } else if(dir->versioning->type == VERSIONING_DELTAV) { DavResource *versions = dav_versiontree(res, NULL); DavResource *v = versions; DavBool first = 1; while(v) { if(!first) { putchar('\n'); } char *vname = dav_get_string_property(v, "D:version-name"); print_resource_version(v, vname); first = 0; v = v->next; } } free(file.path); dav_session_destroy(sn); return ret; } int cmd_trash_info(CmdArgs *a) { if(a->argc != 1) { fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many"); return -1; } SyncDirectory *syncdir = scfg_get_dir(a->argv[0]); if(!syncdir) { fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]); return -1; } if(scfg_check_dir(syncdir)) { return -1; } if(!syncdir->trash) { printf("trash not configured for %s\n", syncdir->name); return 0; } SYS_DIR dir = sys_opendir(syncdir->trash); if(!dir) { fprintf(stderr, "cannot open trash directory: %s\n", syncdir->trash); perror("opendir"); return -1; } uint64_t trashsize = 0; int count = 0; SysDirEnt *ent; while((ent = sys_readdir(dir)) != NULL) { if(!strcmp(ent->name, ".") || !strcmp(ent->name, "..")) { continue; } char *path = util_concat_path(syncdir->trash, ent->name); SYS_STAT s; if(sys_stat(path, &s)) { perror("stat"); } else { trashsize += s.st_size; } count++; free(path); } sys_closedir(dir); printf("path: %s\n", syncdir->trash); printf("%d %s\n", count, count == 1 ? "file" : "files"); char *sizestr = size_str(trashsize); printf("%s\n", sizestr); free(sizestr); return 0; } int cmd_empty_trash(CmdArgs *a) { if(a->argc != 1) { fprintf(stderr, "Too %s arguments\n", a->argc < 1 ? "few" : "many"); return -1; } SyncDirectory *syncdir = scfg_get_dir(a->argv[0]); if(!syncdir) { fprintf(stderr, "Unknown sync dir: %s\n", a->argv[0]); return -1; } if(!syncdir->trash) { fprintf(stderr, "trash not configured for %s\n", syncdir->name); return -1; } SYS_DIR dir = sys_opendir(syncdir->trash); if(!dir) { fprintf(stderr, "cannot open trash directory: %s\n", syncdir->trash); perror("opendir"); return -1; } SysDirEnt *ent; while((ent = sys_readdir(dir)) != NULL) { if(!strcmp(ent->name, ".") || !strcmp(ent->name, "..")) { continue; } char *path = util_concat_path(syncdir->trash, ent->name); printf("delete: %s\n", path); SYS_STAT s; if(sys_stat(path, &s)) { perror("stat"); free(path); continue; } if(S_ISDIR(s.st_mode)) { if(rmdir(path)) { perror("rmdir"); } } else { if(sys_unlink(path)) { perror("unlink"); } } free(path); } sys_closedir(dir); return 0; } #define CMD_TAG_ADD 0 #define CMD_TAG_REMOVE 1 #define CMD_TAG_SET 2 #define CMD_TAG_LIST 3 int cmd_add_tag(CmdArgs *args) { if(args->argc != 2) { fprintf(stderr, "Too %s arguments\n", args->argc <= 1 ? "few" : "many"); return -1; } return cmd_tagop(args, CMD_TAG_ADD); } int cmd_remove_tag(CmdArgs *args) { if(args->argc != 2) { fprintf(stderr, "Too %s arguments\n", args->argc <= 1 ? "few" : "many"); return -1; } return cmd_tagop(args, CMD_TAG_REMOVE); } int cmd_set_tags(CmdArgs *args) { if(args->argc < 1 || args->argc > 2) { fprintf(stderr, "Too %s arguments\n", args->argc < 1 ? "few" : "many"); return -1; } return cmd_tagop(args, CMD_TAG_SET); } int cmd_list_tags(CmdArgs *args) { if(args->argc != 1) { fprintf(stderr, "Too %s arguments\n", args->argc <= 1 ? "few" : "many"); return -1; } return cmd_tagop(args, CMD_TAG_LIST); } int cmd_tagop(CmdArgs *args, int cmd) { SyncFile file; int ret = 0; char *path = args->argv[0]; int err = sync_get_file(args, path, &file, TRUE); if(err) { sync_print_get_file_err(path, err); return -1; } if(!file.dir->tagconfig) { fprintf(stderr, "Tags are not supported for this sync directory\n"); return -1; } SyncDatabase *db = load_db(file.dir->database); if(!db) { fprintf(stderr, "Cannot load sync directory database\n"); return -1; } LocalResource *localres = ucx_map_cstr_get(db->resources, file.path); UcxList *tags = NULL; DavBool store_tags = FALSE; if(cmd != CMD_TAG_SET) { char *tag = args->argv[1]; char *tagcolor = NULL; // TODO: get color tags = sync_get_file_tags(file.dir, localres, NULL, NULL); UcxList *x = NULL; UCX_FOREACH(elm, tags) { DavTag *t = elm->data; if(cmd == CMD_TAG_LIST) { printf("%s\n", t->name); } else if(!strcmp(t->name, tag)) { x = elm; break; } } if(cmd == CMD_TAG_ADD) { if(!x) { DavTag *newtag = malloc(sizeof(DavTag)); newtag->name = tag; newtag->color = tagcolor; tags = ucx_list_append(tags, newtag); store_tags = TRUE; } } else if(cmd == CMD_TAG_REMOVE) { if(tags) { tags = ucx_list_remove(tags, x); } store_tags = TRUE; } } else { if(args->argc == 2) { char *tags_str = args->argv[1]; tags = parse_csv_taglist(tags_str, strlen(tags_str)); store_tags = TRUE; // TODO: read from stdin if tags_str is "-" } else if (args->argc == 1) { store_tags = TRUE; } else { fprintf(stderr, "Too many arguments\n"); ret = -1; } } if(store_tags) { if(sync_store_tags_local(file.dir, NULL, path, tags)) { fprintf(stderr, "Cannot store tags\n"); } if(localres) { localres->tags_updated = TRUE; if(!tags) { if(localres->tags_hash) { free(localres->tags_hash); localres->tags_hash = NULL; } localres->tags_hash = NULL; } } } // store db if(store_db(db, file.dir->database, file.dir->db_settings)) { fprintf(stderr, "Cannot store sync db\n"); ret = -2; } free(file.path); return ret; } int isfileindir(SyncDirectory *dir, const char *path, SyncFile *f) { char *fullpath; if(path[0] != '/') { size_t wdlen = 256; char *wd = malloc(wdlen); while(!getcwd(wd, wdlen)) { if(errno == ERANGE) { wdlen *= 2; char *newbuf = realloc(wd, wdlen); if (newbuf) { wd = newbuf; } else { free(wd); return 0; } } else { free(wd); return 0; } } fullpath = util_concat_path(wd, path); free(wd); } else { fullpath = strdup(path); } // TODO: normalize path DavBool not_in_dir = 0; scstr_t fp = scstr(fullpath); scstr_t dp = scstr(dir->path); if(fp.length == dp.length) { if(sstrcmp(fp, dp)) { not_in_dir = 1; } } else if(fp.length < dp.length) { not_in_dir = 1; } else { if(!sstrprefix(fp, dp)) { not_in_dir = 1; } else { if(dp.ptr[dp.length-1] == '/') { dp.length--; } if(fp.ptr[dp.length] != '/') { not_in_dir = 1; } } } if(not_in_dir) { free(fullpath); return 0; } // TODO: check filter f->dir = dir; f->path = util_concat_path("/", fullpath + strlen(dir->path)); free(fullpath); return 1; } int sync_get_file(CmdArgs *args, const char *path, SyncFile *f, DavBool dostat) { if(dostat) { SYS_STAT s; if(sys_stat(path, &s)) { switch(errno) { case EACCES: return 2; case ENOENT: return 1; default: return 3; } } } char *sdir = cmd_getoption(args, "syncdir"); if(sdir) { SyncDirectory *dir = scfg_get_dir(sdir); if(!dir) { return 6; } if(!isfileindir(dir, path, f)) { return 4; } } else { SyncDirectory *target = NULL; UcxMapIterator i = scfg_directory_iterator(); UcxKey k; SyncDirectory *dir; UCX_MAP_FOREACH(key, dir, i) { if(isfileindir(dir, path, f)) { if(target) { return 5; } else { target = dir; } } } if(!target) { return 4; } } return 0; } void sync_print_get_file_err(const char *path, int err) { switch(err) { case 1: fprintf(stderr, "File %s: not found\n", path); break; case 2: fprintf(stderr, "File %s: permission denied\n", path); break; case 3: fprintf(stderr, "File %s: stat failed: %s\n", path, strerror(errno)); break; case 4: fprintf(stderr, "File %s is not in any syncdir\n", path); break; case 5: fprintf(stderr, "File %s is in multiple syncdirs\n", path); break; case 6: fprintf(stderr, "Syncdir not found\n"); break; } } int cmd_add_directory(CmdArgs *args) { if(!get_repositories()) { fprintf(stderr, "No repositories available. Run 'dav add-repository' first.\n"); fprintf(stderr, "Abort\n"); return -1; } printf("Each sync directory must have an unique name.\n"); char *name = assistant_getcfg("name"); if(!name) { fprintf(stderr, "Abort\n"); return -1; } if(scfg_get_dir(name)) { fprintf(stderr, "Directory %s already exists.\nAbort\n", name); return -1; } printf("Enter local directory path.\n"); char *path = assistant_getcfg("path"); if(!path) { fprintf(stderr, "Abort\n"); return -1; } printf("Specify webdav repository.\n"); UcxList *repos = get_repositories(); int i = 0; UCX_FOREACH(elm, repos) { Repository *r = elm->data; printf("%d) %s\n", i, r->name); i++; } char *repository = assistant_getcfg("repository"); char *reponame = NULL; if(!repository) { fprintf(stderr, "Abort\n"); return -1; } int64_t reponum = 0; if(util_strtoint(repository, &reponum)) { if(reponum < 0) { fprintf(stderr, "Wrong input.\nAbort\n"); return -1; } UcxList *elm = ucx_list_get(repos, reponum); if(elm) { Repository *r = elm->data; reponame = r->name; } else { fprintf(stderr, "Wrong input.\nAbort\n"); return -1; } } else { if(get_repository(sstr(repository))) { reponame = repository; } else { fprintf(stderr, "Repository %s doesn't exist.\nAbort\n", repository); return -1; } } printf("Enter collection relative to the repository base url.\n"); char *collection = assistant_getdefcfg("collection", "/"); char *db = generate_db_name(name); SyncDirectory dir; memset(&dir, 0, sizeof(SyncDirectory)); dir.name = name; dir.path = path; dir.repository = reponame; dir.collection = collection; dir.trash = ".trash"; dir.database = db; int ret = 0; if(add_directory(&dir)) { fprintf(stderr, "Cannot write sync.xml\n"); ret = -1; } else { printf("\nAdded directory: %s (%s)\n", name, path); } free(name); free(path); free(repository); free(collection); free(db); return ret; } int cmd_list_dirs() { UcxMapIterator iter = scfg_directory_iterator(); SyncDirectory *dir; UCX_MAP_FOREACH(key, dir, iter) { printf("%s\n", dir->name); } return 0; } int cmd_check_repositories() { int ret = EXIT_SUCCESS; UcxList *reponames = NULL; { UcxMapIterator iter = scfg_directory_iterator(); SyncDirectory *dir; UCX_MAP_FOREACH(key, dir, iter) { reponames = ucx_list_append(reponames, dir->repository); } } UCX_FOREACH(listelem, reponames) { char *reponame = listelem->data; printf("Checking %s... ", reponame); Repository* repo = get_repository(sstr(reponame)); if (!repo) { printf(" not found in config.xml!\n"); ret = EXIT_FAILURE; } else { DavSession *sn = create_session(ctx, repo, repo->url); if (sn) { DavResource *res = dav_query(sn, "select - from / with depth = 0"); if (res) { printf("OK.\n"); dav_resource_free(res); } else { printf("unavailable!\n"); ret = EXIT_FAILURE; } dav_session_destroy(sn); } else { printf("cannot create session!\n"); ret = EXIT_FAILURE; } } } ucx_list_free(reponames); return ret; } char* create_locktoken_file(const char *syncdirname, const char *locktoken) { sstr_t fname = ucx_sprintf("locktoken-%s.txt", syncdirname); char *path = config_file_path(fname.ptr); free(fname.ptr); FILE *file = sys_fopen(path, "w"); if(file) { fprintf(file, "%s\n", locktoken); fclose(file); return path; } else { perror("Cannot create locktoken file"); free(path); return NULL; } } char* sync_get_content_hash(DavResource *res) { uint32_t flags = res->session->flags; if((flags & DAV_SESSION_ENCRYPT_CONTENT) == DAV_SESSION_ENCRYPT_CONTENT) { char *enc_hash = dav_get_string_property_ns(res, DAV_NS, "crypto-hash"); char *keyname = dav_get_string_property_ns(res, DAV_NS, "crypto-key"); if(enc_hash && keyname) { DavKey *key = dav_context_get_key(res->session->context, keyname); if(!key) { return NULL; } size_t len = 0; char *dec_hash = aes_decrypt(enc_hash, &len, key); if(!dec_hash) { return NULL; } char *hex_hash = util_hexstr((unsigned char*)dec_hash, len); free(dec_hash); return hex_hash; } } else { return dav_get_string_property_ns(res, DAV_NS, "content-hash"); } return NULL; } void sync_set_content_hash(DavResource *res, const unsigned char *hashdata) { uint32_t flags = res->session->flags; if((flags & DAV_SESSION_ENCRYPT_CONTENT) == DAV_SESSION_ENCRYPT_CONTENT) { if(res->session->key) { char *enc_hash = aes_encrypt((const char*)hashdata, DAV_SHA256_DIGEST_LENGTH, res->session->key); if(enc_hash) { dav_set_string_property_ns(res, DAV_NS, "crypto-hash", enc_hash); free(enc_hash); } } } else { char *hex_hash = util_hexstr(hashdata, DAV_SHA256_DIGEST_LENGTH); dav_set_string_property_ns(res, DAV_NS, "content-hash", hex_hash); free(hex_hash); } }