changeset
dav-sync push refactoring: create list of changes before executing sync
Sun, 09 Dec 2018 18:32:14 +0100
- author
- Olaf Wintermann <olaf.wintermann@gmail.com>
- date
- Sun, 09 Dec 2018 18:32:14 +0100
- changeset 501
- 868da3f76267
- parent 500
- 0fe1514667e6
- child 502
- a23fedac340c
dav-sync push refactoring: create list of changes before executing sync
| dav/sync.c | file | annotate | diff | comparison | revisions |
--- a/dav/sync.c Fri Dec 07 11:48:55 2018 +0100 +++ b/dav/sync.c Sun Dec 09 18:32:14 2018 +0100 @@ -1087,180 +1087,206 @@ int sync_skipped = 0; int sync_error = 0; + UcxList *ls_put = NULL; + UcxList *ls_conflict = NULL; + UcxList *ls_update = NULL; + UcxList *ls_delete = NULL; + // upload all changed files //UcxList *resources = cmd_getoption(a, "read") ? // read_changes(dir, db) : local_scan(dir, db); UcxList *resources = local_scan(dir, db); - - UcxMap *lclres = ucx_map_new(db->resources->count); - int ret = 0; + UcxMap *resources_map = ucx_map_new(ucx_list_size(resources)+16); + UCX_FOREACH(elm, resources) { LocalResource *local_res = elm->data; - int error = 0; - DavBool res_filtered = FALSE; + // ignore all files, that are excluded by a static filter (sync.xml) + + // static include/exclude filter if(res_matches_filter(dir, local_res->path+1)) { - res_filtered = TRUE; - } else { - UCX_FOREACH(elm, dir->tagfilter) { - SyncTagFilter *tf = elm->data; - if(!localres_matches_tags(dir, local_res, tf)) { - res_filtered = TRUE; - break; - } + continue; + } + // static tag filter + UCX_FOREACH(elm, dir->tagfilter) { + SyncTagFilter *tf = elm->data; + if(!localres_matches_tags(dir, local_res, tf)) { + continue; } } - if (res_filtered) { - local_res->keep = TRUE; - } else if (!localres_matches_tags(dir, local_res, tagfilter)) { + + // we need a fast file lookup map later + ucx_map_cstr_put(resources_map, local_res->path, local_res); + + // dynamic tag filter + if(!localres_matches_tags(dir, local_res, tagfilter)) { if(!remove_file) { - local_res->keep = TRUE; + LocalResource *dbres = ucx_map_cstr_get( + db->resources, + local_res->path); + if(dbres) { + // this makes sure the file will not be deleted later + dbres->keep = TRUE; + } } - } else { - if(sync_shutdown) { - LocalResource *lr = ucx_map_cstr_remove(db->resources, local_res->path); - if(lr) { - local_res->size = lr->size; - local_res->last_modified = lr->last_modified; - local_res->etag = lr->etag ? strdup(lr->etag) : NULL; - local_resource_free(lr); - ucx_map_cstr_put(lclres, local_res->path, local_res); - } - elm->data = NULL; - continue; - } - - - if(res_isconflict(db, local_res)) { - printf("skip: %s\n", local_res->path); - sync_skipped++; - continue; - } - - // upload every changed file - int is_changed = local_resource_is_changed( + continue; + } + + if(res_isconflict(db, local_res)) { + ls_conflict = ucx_list_append(ls_conflict, local_res); + continue; + } + + int is_changed = local_resource_is_changed( dir, db, local_res, svrres, restore_removed, restore_modified); - if (is_changed || local_res->tags_updated) { - DavResource *res = dav_resource_new(sn, local_res->path); - if(!res) { - print_resource_error(sn, local_res->path); - ret = -1; - sync_error++; - } - - if(local_res->isdirectory) { - dav_exists(res); - if(sn->error == DAV_NOT_FOUND) { - int abort = 0; - // make sure to store tags for newly created cols - local_res->tags_updated = 1; - // create collection - // TODO: show 405 - printf("mkcol: %s\n", local_res->path); - if(sync_mkdir(dir, res, local_res) && sn->error != DAV_METHOD_NOT_ALLOWED) { - print_resource_error(sn, res->path); - ret = -1; - sync_error++; - error = 1; - abort = 1; - } - - if(dir->tagconfig && local_res->tags_updated && !abort) { - sync_update_tags(dir, sn, res, local_res); - } - } else if(sn->error != DAV_OK) { - // dav_exists() failed - print_resource_error(sn, local_res->path); - ret = -1; - sync_error++; - error = 1; - } - } else if(!is_changed) { - if(dir->tagconfig && local_res->tags_updated) { - sync_update_tags(dir, sn, res, local_res); - } - } else { - if(cdt && remote_resource_is_changed(sn, dir, db, local_res)) { - printf("conflict: %s\n", local_res->path); - local_res->last_modified = 0; - local_res->skipped = TRUE; - sync_skipped++; - } else { - printf("put: %s\n", local_res->path); - if(sync_put_resource(dir, res, local_res, &sync_success)) { - sync_error++; - print_resource_error(sn, res->path); - ret = -1; - error = 1; - } - } - } - dav_resource_free(res); + if(is_changed) { + ls_put = ucx_list_append(ls_put, local_res); + } else if(local_res->tags_updated) { + ls_update = ucx_list_append(ls_update, local_res); + } + } + + // find all deleted files and cleanup the database + UcxMapIterator i = ucx_map_iterator(db->resources); + LocalResource *local; + UCX_MAP_FOREACH(key, local, i) { + // all filtered files should be removed from the database + if(res_matches_filter(dir, local->path+1)) { + ucx_map_cstr_remove(db->resources, local->path); + continue; + } + UCX_FOREACH(elm, dir->tagfilter) { + SyncTagFilter *tf = elm->data; + if(!localres_matches_tags(dir, local, tf)) { + ucx_map_cstr_remove(db->resources, local->path); + continue; } } - // remove every locally available resource from db->resource - // the remaining elements are all deleted files - elm->data = NULL; - if(!error) { - ucx_map_cstr_put(lclres, local_res->path, local_res); - } - - LocalResource *lr = ucx_map_cstr_remove(db->resources, local_res->path); - if(lr) { - local_resource_free(lr); + if(!ucx_map_get(resources_map, key)) { + // The current LocalResource is in the database but doesn't exist + // in the filesystem anymore. This means the file was deleted + // and should be deleted on server + if(!archive) { + ls_delete = ucx_list_append(ls_delete, local); + } else { + ucx_map_cstr_remove(db->resources, local->path); + } } } - // delete all removed files - if(ret == 0 && !archive) { - UcxList *cols = NULL; + ls_delete = ucx_list_sort(ls_delete, (cmp_func)resource_pathlen_cmp, NULL); + + // + // BEGIN PUSH + // + + // upload changed files + int ret = 0; + int error = 0; + for(UcxList *elm=ls_put;elm && !sync_shutdown;elm=elm->next) { + LocalResource *local_res = elm->data; + + DavResource *res = dav_resource_new(sn, local_res->path); + if(!res) { + print_resource_error(sn, local_res->path); + ret = -1; + sync_error++; + } - UcxMapIterator i = ucx_map_iterator(db->resources); - LocalResource *local; - UCX_MAP_FOREACH(key, local, i) { - if (!local->keep && !res_matches_filter(dir, local->path+1)) { - if(sync_shutdown) { - ucx_map_cstr_put(lclres, local->path, local_resource_copy(local)); - } else if(sync_delete_remote_resource(dir, sn, local, &sync_delete, &cols)) { - ucx_map_cstr_put(lclres, local->path, local_resource_copy(local)); - if(sn->error != DAV_NOT_FOUND) { - print_resource_error(sn, local->path); - sync_error++; - break; - } + if(local_res->isdirectory) { + dav_exists(res); + if(sn->error == DAV_NOT_FOUND) { + int abort = 0; + // make sure to store tags for newly created cols + local_res->tags_updated = 1; + // create collection + // TODO: show 405 + printf("mkcol: %s\n", local_res->path); + if(sync_mkdir(dir, res, local_res) && sn->error != DAV_METHOD_NOT_ALLOWED) { + print_resource_error(sn, res->path); + ret = -1; + sync_error++; + error = 1; + abort = 1; + } + + if(dir->tagconfig && local_res->tags_updated && !abort) { + sync_update_tags(dir, sn, res, local_res); + } + } else if(sn->error != DAV_OK) { + // dav_exists() failed + print_resource_error(sn, local_res->path); + ret = -1; + sync_error++; + error = 1; + } + } else { + if(cdt && remote_resource_is_changed(sn, dir, db, local_res)) { + printf("conflict: %s\n", local_res->path); + local_res->last_modified = 0; + local_res->skipped = TRUE; + sync_skipped++; + } else { + printf("put: %s\n", local_res->path); + if(sync_put_resource(dir, res, local_res, &sync_success)) { + sync_error++; + print_resource_error(sn, res->path); + ret = -1; + error = 1; } } } + dav_resource_free(res); - cols = ucx_list_sort(cols, (cmp_func)resource_pathlen_cmp, NULL); + LocalResource *dbres = ucx_map_cstr_remove(db->resources, local_res->path); + ucx_map_cstr_put(db->resources, local_res->path, local_res); + //if(dbres) local_resource_free(dbres); + } + + // metadata updates + for(UcxList *elm=ls_update;elm && !sync_shutdown;elm=elm->next) { + LocalResource *local_res = elm->data; + + DavResource *res = dav_resource_new(sn, local_res->path); - UCX_FOREACH(elm, cols) { - local = elm->data; - if (!local->keep && !res_matches_filter(dir, local->path+1)) { - if(sync_shutdown) { - ucx_map_cstr_put(lclres, local->path, local_resource_copy(local)); - } else if(sync_delete_remote_resource(dir, sn, local, &sync_delete, NULL)) { - ucx_map_cstr_put(lclres, local->path, local_resource_copy(local)); - if(sn->error != DAV_NOT_FOUND) { - print_resource_error(sn, local->path); - sync_error++; - break; - } + if(dir->tagconfig && local_res->tags_updated) { + sync_update_tags(dir, sn, res, local_res); + } + } + + // delete all removed files + UcxList *cols = NULL; + UcxList **col_list = &cols; + UcxList *deletelist = ls_delete; + for(int i=0;i<2;i++) { + // the first iteration deletes everything from ls_delete except + // all collections, which are stored in cols + // in the second run all collections will be deleted + for(UcxList *elm=deletelist;elm && !sync_shutdown;elm=elm->next) { + LocalResource *local = elm->data; + if(local->keep) { + continue; + } + if(sync_delete_remote_resource(dir, sn, local, &sync_delete, col_list)) { + if(sn->error != DAV_NOT_FOUND) { + print_resource_error(sn, local->path); + sync_error++; + break; } + } else { + LocalResource *dbres = ucx_map_cstr_remove(db->resources, local->path); + //local_resource_free(dbres); } } - - ucx_list_free(cols); - } - ucx_map_free_content(db->resources, (ucx_destructor)local_resource_free); - ucx_map_free(db->resources); - db->resources = lclres; + deletelist = cols; + col_list = NULL; + } // unlock repository if(locked) { @@ -1283,15 +1309,10 @@ remove(locktokenfile); } + //ucx_map_free_content(db->resources, (ucx_destructor)local_resource_free); + //ucx_map_free(db->resources); + dav_session_destroy(sn); - while(resources) { - UcxList *next = resources->next; - if(resources->data) { - local_resource_free(resources->data); - } - free(resources); - resources = next; - } // Report if(ret != -2) {
