changeset
dav-sync pull refactoring: create list of changes before sync
Fri, 25 Jan 2019 17:01:59 +0100
- author
- Olaf Wintermann <olaf.wintermann@gmail.com>
- date
- Fri, 25 Jan 2019 17:01:59 +0100
- changeset 503
- f84e64afee61
- parent 502
- a23fedac340c
- child 504
- bf3695fee719
dav-sync pull refactoring: create list of changes before sync
| dav/sopt.h | file | annotate | diff | comparison | revisions | |
| dav/sync.c | file | annotate | diff | comparison | revisions | |
| dav/sync.h | file | annotate | diff | comparison | revisions |
--- a/dav/sopt.h Tue Jan 08 16:37:13 2019 +0100 +++ b/dav/sopt.h Fri Jan 25 17:01:59 2019 +0100 @@ -35,6 +35,8 @@ extern "C" { #endif +typedef void* ArgBool; + typedef struct { UcxMap *options; char **argv;
--- a/dav/sync.c Tue Jan 08 16:37:13 2019 +0100 +++ b/dav/sync.c Fri Jan 25 17:01:59 2019 +0100 @@ -525,12 +525,16 @@ int sync_delete = 0; int sync_error = 0; - UcxList *ls_get = NULL; - UcxList *ls_remove = NULL; - UcxList *ls_conflict = NULL; - UcxList *ls_broken = NULL; + UcxList *res_modified = NULL; + UcxList *res_new = NULL; + UcxList *res_conflict = NULL; + UcxList *res_mkdir = NULL; + UcxList *res_metadata = NULL; + UcxList *res_broken = NULL; + UcxList *lres_removed = NULL; // list of LocalResource* UcxMap *svrres = ucx_map_new(db->resources->count); + UcxMap *dbres = ucx_map_clone(db->resources, NULL, NULL); UcxList *statls = NULL; @@ -571,28 +575,41 @@ if(status && !strcmp(status, "broken")) { res = res->next; localres_keep(db, res->path); - ls_broken = ucx_list_append(ls_broken, res); + res_broken = ucx_list_append(res_broken, res); continue; } - // download the resource - if(!sync_shutdown && sync_get_resource(a, dir, res, db, FALSE, &sync_success)) { - fprintf(stderr, "sync_get_resource failed for resource: %s\n", res->path); - sync_error++; + // check if a resource has changed on the server + int change = resource_get_remote_change(a, res, dir, db); + switch(change) { + case REMOTE_NO_CHANGE: break; + case REMOTE_CHANGE_MODIFIED: { + res_modified = ucx_list_append(res_modified, res); + break; + } + case REMOTE_CHANGE_NEW: { + res_new = ucx_list_append(res_new, res); + break; + } + case REMOTE_CHANGE_DELETED: break; // never happens + case REMOTE_CHANGE_CONFLICT_LOCAL_MODIFIED: { + res_conflict = ucx_list_append(res_conflict, res); + break; + } + case REMOTE_CHANGE_METADATA: { + res_metadata = ucx_list_append(res_metadata, res); + break; + } + case REMOTE_CHANGE_MKDIR: { + res_mkdir = ucx_list_append(res_mkdir, res); + break; + } } - // add every resource from the server to svrres - // then db-resources contains only resources which are not on the - // server - LocalResource *local = ucx_map_cstr_remove(db->resources, res->path); - if(local) { - ucx_map_cstr_put(svrres, res->path, local); - - if(local->last_modified == 0) { - // stat this file later (directory) - statls = ucx_list_prepend(statls, local); - } - } // else: sync_shutdown is TRUE + // remove every server resource from dbres + // all remaining elements are the resources that are removed + // on the server + ucx_map_cstr_remove(dbres, res->path); if(res->children) { stack = ucx_list_prepend(stack, res->children); @@ -601,46 +618,109 @@ } } - // stat all files with unknown lastmodified date - UCX_FOREACH(elm, statls) { - LocalResource *l = elm->data; - char *local_path = util_concat_path(dir->path, l->path); - SYS_STAT s; - if(!sys_stat(local_path, &s)) { - l->last_modified = s.st_mtime; - } - free(local_path); - } - ucx_list_free(statls); - - // delete every remotely removed resource - UcxMapIterator i = ucx_map_iterator(db->resources); + // find deleted resources + // svrres currently contains all resources from the server + // and will replace the current db->resources map later + UcxMapIterator i = ucx_map_iterator(dbres); LocalResource *local; - UcxList *rmdirs = NULL; UCX_MAP_FOREACH(key, local, i) { if (res_matches_filter(dir, local->path)) { continue; } + if(!local->keep) { + lres_removed = ucx_list_prepend(lres_removed, local); + } + } + + // the first thing we need are all directories to put the files in + UCX_FOREACH(elm, res_mkdir) { + DavResource *res = elm->data; + char *local_path = util_concat_path(dir->path, res->path); + if(sys_mkdir(local_path) && errno != EEXIST) { + fprintf(stderr, + "Cannot create directory %s: %s", + local_path, strerror(errno)); + } + free(local_path); + } + + // we need a map for all conflicts for fast lookups + UcxMap *conflicts = ucx_map_new(ucx_list_size(res_conflict)+16); + UCX_FOREACH(elm, res_conflict) { + DavResource *res = elm->data; + ucx_map_cstr_put(conflicts, res->path, res); + } + + // download all new, modified and conflict files + UcxList *download = ucx_list_concat(res_modified, res_conflict); + download = ucx_list_concat(res_new, download); + UCX_FOREACH(elm, download) { + DavResource *res = elm->data; + if(sync_shutdown) { + break; + } - if(sync_shutdown || local->keep) { - ucx_map_cstr_put(svrres, local->path, local_resource_copy(local)); - } else { - // sync_remove_resource does all necessary tests - int ret = sync_remove_local_resource(dir, local); - if(ret == -1) { - rmdirs = ucx_list_append(rmdirs, local); - } else if(ret == 0) { - sync_delete++; - } + if(ucx_map_cstr_get(conflicts, res->path)) { + rename_conflict_file(dir, db, res->path); + } + + // download the resource + if(sync_get_resource(a, dir, res, db, &sync_success)) { + fprintf(stderr, "resource download failed: %s\n", res->path); + sync_error++; } } + + UCX_FOREACH(elm, res_metadata) { + DavResource *res = elm->data; + if(sync_shutdown) { + break; + } + + LocalResource *local = ucx_map_cstr_get(db->resources, res->path); + if(local) { + char *local_path = util_concat_path(dir->path, res->path); + if(sync_store_tags(dir, local_path, local, res)) { + fprintf(stderr, "Tag update failed: %s\n", res->path); + } + free(local_path); + } else { + // this should never happen but who knows + fprintf(stderr, + "Cannot update metadata of file %s: not in database\n", + res->path); + } + } + + UcxList *rmdirs = NULL; + UCX_FOREACH(elm, lres_removed) { + LocalResource *res = elm->data; + if(sync_shutdown) { + break; + } + + int ret = sync_remove_local_resource(dir, res); + if(ret == -1) { + rmdirs = ucx_list_append(rmdirs, res); + } else if(ret == 0) { + LocalResource *local = ucx_map_cstr_remove(db->resources, res->path); + if(local) { + local_resource_free(local); + } + sync_delete++; + } + } + UCX_FOREACH(elm, rmdirs) { LocalResource *local_dir = elm->data; - sync_remove_local_directory(dir, local_dir); + if(!sync_remove_local_directory(dir, local_dir)) { + LocalResource *local = ucx_map_cstr_remove(db->resources, local_dir->path); + if(local) { + local_resource_free(local); + } + sync_delete++; + } } - ucx_map_free_content(db->resources, (ucx_destructor)local_resource_free); - ucx_map_free(db->resources); - db->resources = svrres; // unlock repository if(locked) { @@ -679,26 +759,68 @@ return ret; } -int sync_remote_is_changed( + +RemoteChangeType resource_get_remote_change( CmdArgs *a, - SyncDirectory *dir, DavResource *res, - SyncDatabase *db, - DavBool force, - DavBool *conflict, - DavBool *metadataupdate) + SyncDirectory *dir, + SyncDatabase *db) { - int cdt = cmd_getoption(a, "conflict") ? 0 : 1; // conflict detection - *conflict = 0; - *metadataupdate = 0; + char *etag = dav_get_string_property(res, "D:getetag"); + if(!etag) { + fprintf(stderr, "Error: resource %s has no etag\n", res->path); + return REMOTE_NO_CHANGE; + } + + RemoteChangeType type = cmd_getoption(a, "conflict") ? + REMOTE_CHANGE_MODIFIED : REMOTE_CHANGE_CONFLICT_LOCAL_MODIFIED; LocalResource *local = ucx_map_cstr_get(db->resources, res->path); char *local_path = util_concat_path(dir->path, res->path); - char *etag = dav_get_string_property(res, "D:getetag"); SYS_STAT s; - int ret = 0; + DavBool exists = 1; + if(sys_stat(local_path, &s)) { + if(errno != ENOENT) { + fprintf(stderr, "Cannot stat file: %s\n", local_path); + free(local_path); + return REMOTE_NO_CHANGE; + } + exists = 0; + } + RemoteChangeType ret = REMOTE_NO_CHANGE; + if(res->iscollection) { + if(!exists) { + ret = REMOTE_CHANGE_MKDIR; + } + } else if(local) { + DavBool nochange = FALSE; + if(local->etag) { + sstr_t e = sstr(etag); + if(sstrprefix(e, S("W/"))) { + e = sstrsubs(e, 2); + } + if(!strcmp(e.ptr, local->etag)) { + // resource is already up-to-date on the client + + // TODO: detect metadata update + //sync_store_tags(dir, local_path, local, res); + nochange = TRUE; + } + } + + if(!nochange) { + if(!(exists && s.st_mtime != local->last_modified)) { + type = REMOTE_CHANGE_MODIFIED; + } + ret = type; + } + } else if(exists) { + ret = type; + } else { + ret = REMOTE_CHANGE_NEW; + } free(local_path); return ret; @@ -709,149 +831,79 @@ SyncDirectory *dir, DavResource *res, SyncDatabase *db, - DavBool force, int *counter) -{ - int cdt = cmd_getoption(a, "conflict") ? 0 : 1; // conflict detection - +{ LocalResource *local = ucx_map_cstr_get(db->resources, res->path); char *local_path = util_concat_path(dir->path, res->path); char *etag = dav_get_string_property(res, "D:getetag"); SYS_STAT s; memset(&s, 0, sizeof(SYS_STAT)); - if(!force) { - if(local && !res->iscollection) { - int exists = 1; - if(sys_stat(local_path, &s)) { - // Ignore the fact, that the file is locally removed. If the - // server has an updated version, we read the file or the - // next push will delete it on the server. - if(errno != ENOENT) { - fprintf(stderr, "Cannot stat file: %s\n", local_path); - free(local_path); - return -1; - } else { - exists = 0; - } - } - - if(local->etag) { - sstr_t e = sstr(etag); - if(sstrprefix(e, S("W/"))) { - e = sstrsubs(e, 2); - } - if(!strcmp(e.ptr, local->etag)) { - // resource is already up-to-date on the client - sync_store_tags(dir, local_path, local, res); - free(local_path); - return 0; - } - } - - if(cdt && exists && s.st_mtime != local->last_modified) { - // file modified on the server and on the client - rename_conflict_file(dir, db, local->path); - } - } else { - if(sys_stat(local_path, &s)) { - if(errno != ENOENT) { - fprintf(stderr, "Cannot stat file: %s\n", local_path); - } - } else if(S_ISDIR(s.st_mode)) { - //fprintf(stderr, "Error: file %s is a directory\n", local_path); - } else if(cdt) { - // rename file on conflict - rename_conflict_file(dir, db, res->path); - } - } - } int ret = 0; char *tmp_path = create_tmp_download_path(local_path); - if(res->iscollection) { - if(sys_mkdir(local_path) && errno != EEXIST) { - ret = -1; + + if(!tmp_path) { + fprintf(stderr, "Cannot create tmp path for %s\n", local_path); + free(local_path); + return -1; + } + FILE *out = sys_fopen(tmp_path, "wb"); + if(!out) { + fprintf(stderr, "Cannot open output file: %s\n", local_path); + free(local_path); + free(tmp_path); + return -1; + } + printf("get: %s\n", res->path); + if(dav_get_content(res, out, (dav_write_func)fwrite)) { + ret = -1; + } + fclose(out); + + if(ret == 0) { + (*counter)++; + + if(dir->trash && dir->backuppull) { + move_to_trash(dir, local_path); } - - if(ret == 0) { - if(!local) { - // new local resource - local = calloc(1, sizeof(LocalResource)); - local->path = util_concat_path(res->path, "/"); - local->last_modified = 0; - if(local->etag) { - free(local->etag); - } - local->etag = strdup(etag); - ucx_map_cstr_put(db->resources, local->path, local); - } - - sync_store_tags(dir, local_path, local, res); - } - } else { - if(!tmp_path) { - fprintf(stderr, "Cannot create tmp path for %s\n", local_path); + if(sys_rename(tmp_path, local_path)) { + fprintf( + stderr, + "Cannot rename file %s to %s\n", + tmp_path, + local_path); + perror(""); + free(tmp_path); free(local_path); return -1; } - FILE *out = sys_fopen(tmp_path, "wb"); - if(!out) { - fprintf(stderr, "Cannot open output file: %s\n", local_path); - free(local_path); - free(tmp_path); - return -1; + + if(sys_stat(local_path, &s)) { + fprintf(stderr, + "Cannot stat file %s: %s\n", local_path, strerror(errno)); } - printf("get: %s\n", res->path); - if(dav_get_content(res, out, (dav_write_func)fwrite)) { - ret = -1; + + if(!local) { + // new local resource + local = calloc(1, sizeof(LocalResource)); + local->path = strdup(res->path); + ucx_map_cstr_put(db->resources, local->path, local); } - fclose(out); + + if(local->etag) { + free(local->etag); + } + // set metadata from stat + local->etag = strdup(etag); + local->last_modified = s.st_mtime; + local->size = s.st_size; + local->skipped = FALSE; sync_store_tags(dir, tmp_path, local, res); - - if(ret == 0) { - (*counter)++; - - if(dir->trash && dir->backuppull) { - move_to_trash(dir, local_path); - } - if(sys_rename(tmp_path, local_path)) { - fprintf( - stderr, - "Cannot rename file %s to %s\n", - tmp_path, - local_path); - perror(""); - free(tmp_path); - free(local_path); - return -1; - } - - if(sys_stat(local_path, &s)) { - fprintf(stderr, "Cannot stat file: %s\n", local_path); - perror(""); - } - - if(!local) { - // new local resource - local = calloc(1, sizeof(LocalResource)); - local->path = strdup(res->path); - ucx_map_cstr_put(db->resources, local->path, local); - } - - if(local->etag) { - free(local->etag); - } - // set metadata from stat - local->etag = strdup(etag); - local->last_modified = s.st_mtime; - local->size = s.st_size; - local->skipped = FALSE; - } else { - if(sys_unlink(tmp_path)) { - fprintf(stderr, "Cannot remove tmp file: %s\n", tmp_path); - } + } else { + if(sys_unlink(tmp_path)) { + fprintf(stderr, "Cannot remove tmp file: %s\n", tmp_path); } } @@ -891,16 +943,18 @@ return ret; } -void sync_remove_local_directory(SyncDirectory *dir, LocalResource *res) { +int sync_remove_local_directory(SyncDirectory *dir, LocalResource *res) { + int ret = 0; char *local_path = util_concat_path(dir->path, res->path); printf("delete: %s\n", res->path); if(rmdir(local_path)) { - fprintf(stderr, "rmdir: %s : ", local_path); - perror(NULL); + fprintf(stderr, "rmdir: %s : %s", local_path, strerror(errno)); + ret = 1; } free(local_path); + return ret; } void rename_conflict_file(SyncDirectory *dir, SyncDatabase *db, char *path) { @@ -1474,7 +1528,7 @@ } // download the resource - if(!sync_shutdown && sync_get_resource(a, dir, res, db, TRUE, &sync_success)) { + if(!sync_shutdown && sync_get_resource(a, dir, res, db, &sync_success)) { fprintf(stderr, "sync_get_resource failed for resource: %s\n", res->path); sync_error++; }
--- a/dav/sync.h Tue Jan 08 16:37:13 2019 +0100 +++ b/dav/sync.h Fri Jan 25 17:01:59 2019 +0100 @@ -59,6 +59,22 @@ SyncDirectory *dir; char *path; } SyncFile; + +enum RemoteChangeType { + REMOTE_NO_CHANGE = 0, + REMOTE_CHANGE_MODIFIED, + REMOTE_CHANGE_NEW, + REMOTE_CHANGE_DELETED, + REMOTE_CHANGE_CONFLICT_LOCAL_MODIFIED, + REMOTE_CHANGE_METADATA, + REMOTE_CHANGE_MKDIR +}; +typedef enum RemoteChangeType RemoteChangeType; + +typedef struct RemoteChange { + DavResource *resource; + RemoteChangeType type; +} RemoteChange; void print_usage(char *cmd); @@ -71,15 +87,20 @@ int cmd_push(CmdArgs *args, DavBool archive); int cmd_restore(CmdArgs *args); +RemoteChangeType resource_get_remote_change( + CmdArgs *a, + DavResource *res, + SyncDirectory *dir, + SyncDatabase *db); + int sync_get_resource( CmdArgs *a, SyncDirectory *dir, DavResource *res, SyncDatabase *db, - DavBool force, int *counter); int sync_remove_local_resource(SyncDirectory *dir, LocalResource *res); -void sync_remove_local_directory(SyncDirectory *dir, LocalResource *res); +int sync_remove_local_directory(SyncDirectory *dir, LocalResource *res); void rename_conflict_file(SyncDirectory *dir, SyncDatabase *db, char *path); char* create_tmp_download_path(char *path); void move_to_trash(SyncDirectory *dir, char *path);
