changeset
improves metadata support in dav-sync push
Sun, 17 Mar 2019 15:00:48 +0100
- author
- Olaf Wintermann <olaf.wintermann@gmail.com>
- date
- Sun, 17 Mar 2019 15:00:48 +0100
- changeset 525
- 26a1d5b9d9d2
- parent 524
- d53fd1006485
- child 526
- e3c0440bd599
improves metadata support in dav-sync push
| dav/db.c | file | annotate | diff | comparison | revisions | |
| dav/db.h | file | annotate | diff | comparison | revisions | |
| dav/sync.c | file | annotate | diff | comparison | revisions | |
| dav/sync.h | file | annotate | diff | comparison | revisions | |
| dav/tags.c | file | annotate | diff | comparison | revisions | |
| dav/tags.h | file | annotate | diff | comparison | revisions | |
| libidav/resource.c | file | annotate | diff | comparison | revisions | |
| libidav/webdav.c | file | annotate | diff | comparison | revisions | |
| libidav/webdav.h | file | annotate | diff | comparison | revisions |
--- a/dav/db.c Fri Mar 15 20:30:09 2019 +0100 +++ b/dav/db.c Sun Mar 17 15:00:48 2019 +0100 @@ -135,8 +135,6 @@ field = 8; } else if(xstreq(name, "remote-tags-hash")) { field = 9; - } else if(xstreq(name, "remote-xattr-hash")) { - field = 10; } else if(xstreq(name, "skipped")) { res->skipped = TRUE; } else if(xstreq(name, "tags-updated")) { @@ -213,10 +211,6 @@ res->remote_tags_hash = strdup((char*)value); break; } - case 10: { - res->remote_xattr_hash = strdup((char*)value); - break; - } } } else if(XML_READER_TYPE_END_ELEMENT) { if(xstreq(name, "resource")) { @@ -400,30 +394,6 @@ } } - if(res->remote_tags_hash) { - r = xmlTextWriterWriteElement( - writer, - BAD_CAST "remote-tags-hash", - BAD_CAST res->remote_tags_hash); - if(r < 0) { - fprintf(stderr, "Cannot write remote-tags-hash: %s\n", res->remote_tags_hash); - xmlFreeTextWriter(writer); - return -1; - } - } - - if(res->remote_xattr_hash) { - r = xmlTextWriterWriteElement( - writer, - BAD_CAST "remote-xattr-hash", - BAD_CAST res->remote_xattr_hash); - if(r < 0) { - fprintf(stderr, "Cannot write remote-tags-hash: %s\n", res->remote_xattr_hash); - xmlFreeTextWriter(writer); - return -1; - } - } - if(res->skipped) { r = xmlTextWriterStartElement(writer, BAD_CAST "skipped"); r += xmlTextWriterEndElement(writer);
--- a/dav/db.h Fri Mar 15 20:30:09 2019 +0100 +++ b/dav/db.h Sun Mar 17 15:00:48 2019 +0100 @@ -65,7 +65,6 @@ char *tags_hash; char *xattr_hash; char *remote_tags_hash; - char *remote_xattr_hash; DavBool tags_updated; DavBool finfo_updated;
--- a/dav/sync.c Fri Mar 15 20:30:09 2019 +0100 +++ b/dav/sync.c Sun Mar 17 15:00:48 2019 +0100 @@ -71,6 +71,22 @@ va_end(ap); } +/* + * strcmp version that works with NULL pointers + */ +static int nullstrcmp(const char *s1, const char *s2) { + if(!s1 && s2) { + return -1; + } + if(s1 && s2) { + return 1; + } + if(!s1 && !s2) { + return 0; + } + return strcmp(s1, s2); +} + int main(int argc, char **argv) { if(argc < 2) { fprintf(stderr, "Missing command\n"); @@ -311,7 +327,7 @@ } DavBool changed = 0; - UcxList *res_tags = sync_get_file_tags(dir, res, &changed); + UcxList *res_tags = sync_get_file_tags(dir, res, &changed, NULL); int ret = matches_tagfilter(res_tags, tagfilter); UCX_FOREACH(elm, res_tags) { @@ -1319,7 +1335,7 @@ abort = 1; } - if(dir->tagconfig && local_res->tags_updated && !abort) { + if(local_res->metadata_updated && !abort) { sync_update_metadata(dir, sn, res, local_res); } } else if(sn->error != DAV_OK) { @@ -1330,7 +1346,7 @@ error = 1; } } else { - if(cdt && remote_resource_is_changed(sn, dir, db, local_res)) { + if(cdt && remote_resource_is_changed(sn, dir, db, res, local_res)) { printf("conflict: %s\n", local_res->path); local_res->last_modified = 0; local_res->skipped = TRUE; @@ -1357,6 +1373,18 @@ LocalResource *local_res = elm->data; DavResource *res = dav_resource_new(sn, local_res->path); + if(dir->tagconfig) { + DavPropName properties[] = { + {DAV_NS,"tags"}, + }; + if(dav_load_prop(res, properties, 1)) { + sync_error++; + print_resource_error(sn, res->path); + ret = -1; + error = 1; + continue; + } + } if(local_res->metadata_updated) { if(!sync_update_metadata(dir, sn, res, local_res)) { @@ -1764,6 +1792,12 @@ if(db_res->tags_hash) { res->tags_hash = strdup(db_res->tags_hash); } + if(db_res->remote_tags_hash) { + res->remote_tags_hash = strdup(db_res->remote_tags_hash); + } + if(db_res->xattr_hash) { + res->xattr_hash = strdup(db_res->xattr_hash); + } if(dir->tagconfig && dir->tagconfig->detect_changes && !res->tags_updated) { UcxBuffer *tags = sync_get_file_tag_data(dir, res); @@ -1823,15 +1857,22 @@ DavSession *sn, SyncDirectory *dir, SyncDatabase *db, + DavResource *remote, LocalResource *res) { + DavPropName properties[] = { + {"DAV:","getetag"}, + {DAV_NS,"tags"}, + {DAV_NS,VERSION_PATH_PROPERTY} + }; + int err = dav_load_prop(remote, properties, 3); + if(res->restore) { return 0; } - DavResource *remote = dav_get(sn, res->path, "D:getetag"); int ret = 0; - if(remote) { + if(err == 0) { char *etag = dav_get_string_property(remote, "D:getetag"); if(!res->etag) { // the resource is on the server and the client has no etag @@ -1848,7 +1889,6 @@ // something weird is happening, the server must support etags fprintf(stderr, "Warning: resource %s has no etag\n", remote->href); } - dav_resource_free(remote); } return ret; } @@ -1882,38 +1922,6 @@ return ret; } -UcxList* sync_merge_tags(UcxList *tags1, UcxList *tags2) { - // this map is used to check the existence of tags - UcxMap *tag_map = ucx_map_new(32); - // merged taglist - UcxList *new_tags = NULL; - - // add all local tags - UCX_FOREACH(elm, tags1) { - DavTag *t = elm->data; - ucx_map_cstr_put(tag_map, t->name, t); - DavTag *newt = calloc(1, sizeof(DavTag)); - newt->color = t->color ? strdup(t->color) : NULL; - newt->name = strdup(t->name); - new_tags = ucx_list_append(new_tags, newt); - } - // check if a remote tag is already in the map - // and if not add it to the new taglist - UCX_FOREACH(elm, tags2) { - DavTag *t = elm->data; - if(!ucx_map_cstr_get(tag_map, t->name)) { - DavTag *newt = calloc(1, sizeof(DavTag)); - newt->color = t->color ? strdup(t->color) : NULL; - newt->name = strdup(t->name); - new_tags = ucx_list_append(new_tags, newt); - } - } - - ucx_map_free(tag_map); - - return new_tags; -} - int sync_tags_equal(UcxList *tags1, UcxList *tags2) { if(!tags1) { return tags2 ? 0 : 1; @@ -1962,7 +1970,7 @@ DavBool store_tags = FALSE; DavBool tags_changed = FALSE; - UcxList *local_tags = sync_get_file_tags(dir, local, &tags_changed); + UcxList *local_tags = sync_get_file_tags(dir, local, &tags_changed, NULL); if(tags_changed) { switch(dir->tagconfig->conflict) { case TAG_NO_CONFLICT: @@ -1975,7 +1983,7 @@ local->tags_updated = FALSE; } case TAG_MERGE: { - UcxList *new_tags = sync_merge_tags(local_tags, tags); + UcxList *new_tags = merge_tags(local_tags, tags); // TODO: free tags and local_tags tags = new_tags; store_tags = TRUE; @@ -2085,7 +2093,7 @@ return buf; } -UcxList* sync_get_file_tags(SyncDirectory *dir, LocalResource *res, DavBool *changed) { +UcxList* sync_get_file_tags(SyncDirectory *dir, LocalResource *res, DavBool *changed, char **newhash) { if(changed) *changed = FALSE; UcxList *tags = NULL; @@ -2106,16 +2114,20 @@ sync_get_file_tag_data(dir, res); if(tag_buf) { - char *newhash = dav_create_hash(tag_buf->space, tag_buf->size); + char *new_hash = dav_create_hash(tag_buf->space, tag_buf->size); if(res->tags_hash) { - if(changed && strcmp(res->tags_hash, newhash)) { + if(changed && strcmp(res->tags_hash, new_hash)) { *changed = TRUE; } free(res->tags_hash); } else { if(changed) *changed = TRUE; } - res->tags_hash = newhash; + if(!newhash) { + *newhash = new_hash; + } else { + free(newhash); + } switch(dir->tagconfig->local_format) { default: break; @@ -2166,10 +2178,10 @@ #define VBEGIN_ERROR_MOVE 2 #define VBEGIN_ERROR_PROPPATCH 3 #define VBEGIN_ERROR_CHECKOUT 4 -int versioning_begin(SyncDirectory *dir, DavResource *res, int *exists) { +int versioning_begin(SyncDirectory *dir, DavResource *res) { int ret = 0; - if(dir->versioning->type == VERSIONING_SIMPLE && *exists) { + if(dir->versioning->type == VERSIONING_SIMPLE && res->exists) { DavResource *history_collection = dav_resource_new( res->session, dir->versioning->collection); @@ -2179,11 +2191,6 @@ // better error handling is done later (sync_put_resource) // if there is no history collection for this resource, we create one - DavPropName prop; - prop.ns = DAV_NS; - prop.name = VERSION_PATH_PROPERTY; - *exists = dav_load_prop(res, &prop, 1); - char *history_href = NULL; char *vcol_path = dav_get_string_property_ns(res, DAV_NS, VERSION_PATH_PROPERTY); if(!vcol_path) { @@ -2289,6 +2296,27 @@ return 0; } +static void update_metadata_hashes(LocalResource *local, MetadataHashes hashes) { + if(hashes.tags) { + if(local->tags_hash) { + free(local->tags_hash); + } + local->tags_hash = hashes.tags; + } + if(hashes.tags_remote) { + if(local->remote_tags_hash) { + free(local->remote_tags_hash); + } + local->remote_tags_hash = hashes.tags_remote; + } + if(hashes.xattr) { + if(local->xattr_hash) { + free(local->xattr_hash); + } + local->xattr_hash = hashes.xattr; + } +} + int sync_put_resource( SyncDirectory *dir, DavResource *res, @@ -2315,24 +2343,19 @@ dav_set_content(res, in, (dav_read_func)myread, (dav_seek_func)file_seek); dav_set_content_length(res, s.st_size); - if(dir->tagconfig) { - UcxList *tags = sync_get_file_tags(dir, local, NULL); - DavXmlNode *prop = create_xml_taglist(tags); - if(prop) { - dav_set_property_ns(res, DAV_NS, "tags", prop); - } - } + MetadataHashes hashes; + hashes = sync_set_metadata_properties(dir, res->session, res, local); - int exists = dav_exists(res); + // before sync_put_resource, remote_resource_is_changed does a propfind + // and sets res->exists + int exists = res->exists; if(dir->versioning && dir->versioning->always) { - int err = versioning_begin(dir, res, &exists); + int err = versioning_begin(dir, res); if(err) { fprintf(stderr, "Cannot store version for resource: %s\n", res->href); free(local_path); return -1; } - } else { - exists = dav_exists(res); } int ret = -1; @@ -2358,6 +2381,8 @@ if(ret == 0) { (*counter)++; + update_metadata_hashes(local, hashes); + // check contentlength and get new etag DavResource *up_res = dav_get(res->session, res->path, "D:getetag,idav:status,idav:tags"); @@ -2483,17 +2508,70 @@ return ret; } -int sync_update_metadata(SyncDirectory *dir, DavSession *sn, DavResource *res, LocalResource *local) { - if(dir->tagconfig && local->tags_updated) { +MetadataHashes sync_set_metadata_properties( + SyncDirectory *dir, + DavSession *sn, + DavResource *res, + LocalResource *local) +{ + MetadataHashes hashes = {NULL, NULL, NULL}; + if(dir->tagconfig) { // get local tags - UcxList *tags = sync_get_file_tags(dir, local, NULL); - - DavXmlNode *prop = create_xml_taglist(tags); - if(prop) { - dav_set_property_ns(res, DAV_NS, "tags", prop); + DavBool changed = 0; + char *tags_hash = NULL; + UcxList *tags = sync_get_file_tags(dir, local, &changed, &tags_hash); + if(changed || local->tags_updated) { + hashes.tags = tags_hash; + + DavBool store_tags = TRUE; + // get remote tags + UcxList *remote_tags = NULL; + DavXmlNode *tagsprop = dav_get_property_ns(res, DAV_NS, "tags"); + if(tagsprop) { + remote_tags = parse_dav_xml_taglist(tagsprop); + } + char *remote_hash = create_tags_hash(remote_tags); + if(nullstrcmp(remote_hash, local->remote_tags_hash)) { + // the tags have changed on the server + switch(dir->tagconfig->conflict) { + case TAG_NO_CONFLICT: break; + case TAG_KEEP_LOCAL: break; + case TAG_KEEP_REMOTE: { + store_tags = FALSE; + local->tags_updated = FALSE; + break; + } + case TAG_MERGE: { + UcxList *new_tags = merge_tags(tags, remote_tags); + free_taglist(tags); + tags = new_tags; + break; + } + } + } + + if(dir->tagconfig->local_format == TAG_FORMAT_CSV) { + // csv tag lists don't have colors, so we have to add + // the colors from the remote tag list + add_tag_colors(tags, remote_tags); + } + + if(store_tags) { + if(tags) { + DavXmlNode *tagprop = create_xml_taglist(tags); + dav_set_property_ns(res, DAV_NS, "tags", tagprop); + } else { + dav_remove_property_ns(res, DAV_NS, "tags"); + } + } + + free_taglist(remote_tags); } else { - dav_remove_property_ns(res, DAV_NS, "tags"); + if(tags_hash) { + free(tags_hash); + } } + free_taglist(tags); } if(local->finfo_updated) { @@ -2508,10 +2586,21 @@ if(local->xattr_updated) { if(local->xattr) { resource_set_xattr(res, local->xattr); + hashes.xattr = strdup(local->xattr->hash); } else { dav_remove_property(res, "idav:xattributes"); } } + return hashes; +} + +int sync_update_metadata( + SyncDirectory *dir, + DavSession *sn, + DavResource *res, + LocalResource *local) +{ + MetadataHashes hashes = sync_set_metadata_properties(dir, sn, res, local); int err = 0; printf("update: %s\n", local->path); @@ -2519,24 +2608,9 @@ print_resource_error(sn, local->path); err = 1; } else { - if(dir->tagconfig && local->tags_updated) { - UcxBuffer *tag_data = local->cached_tags; - if(local->tags_hash) { - free(local->tags_hash); - local->tags_hash = NULL; - } - if(tag_data) { - char *hash = dav_create_hash(tag_data->space, tag_data->size); - local->tags_hash = hash; - } - local->tags_updated = FALSE; - } - if(local->xattr) { - local->xattr_hash = strdup(local->xattr->hash); - } + update_metadata_hashes(local, hashes); } - - // TODO: free stuff + return err; } @@ -2895,7 +2969,7 @@ fprintf(stderr, "Too %s arguments\n", args->argc <= 1 ? "few" : "many"); return -1; } - return cmd_tagopt(args, CMD_TAG_ADD); + return cmd_tagop(args, CMD_TAG_ADD); } int cmd_remove_tag(CmdArgs *args) { @@ -2903,7 +2977,7 @@ fprintf(stderr, "Too %s arguments\n", args->argc <= 1 ? "few" : "many"); return -1; } - return cmd_tagopt(args, CMD_TAG_REMOVE); + return cmd_tagop(args, CMD_TAG_REMOVE); } int cmd_set_tags(CmdArgs *args) { @@ -2911,7 +2985,7 @@ fprintf(stderr, "Too %s arguments\n", args->argc < 1 ? "few" : "many"); return -1; } - return cmd_tagopt(args, CMD_TAG_SET); + return cmd_tagop(args, CMD_TAG_SET); } int cmd_list_tags(CmdArgs *args) { @@ -2919,10 +2993,10 @@ fprintf(stderr, "Too %s arguments\n", args->argc <= 1 ? "few" : "many"); return -1; } - return cmd_tagopt(args, CMD_TAG_LIST); + return cmd_tagop(args, CMD_TAG_LIST); } -int cmd_tagopt(CmdArgs *args, int cmd) { +int cmd_tagop(CmdArgs *args, int cmd) { SyncFile file; int ret = 0; char *path = args->argv[0]; @@ -2952,7 +3026,7 @@ char *tag = args->argv[1]; char *tagcolor = NULL; // TODO: get color - tags = sync_get_file_tags(file.dir, localres, NULL); + tags = sync_get_file_tags(file.dir, localres, NULL, NULL); UcxList *x = NULL; UCX_FOREACH(elm, tags) { DavTag *t = elm->data;
--- a/dav/sync.h Fri Mar 15 20:30:09 2019 +0100 +++ b/dav/sync.h Sun Mar 17 15:00:48 2019 +0100 @@ -60,6 +60,12 @@ char *path; } SyncFile; +typedef struct MetadataHashes { + char *tags; + char *tags_remote; + char *xattr; +} MetadataHashes; + enum RemoteChangeType { REMOTE_NO_CHANGE = 0, REMOTE_CHANGE_MODIFIED, @@ -119,6 +125,7 @@ DavSession *sn, SyncDirectory *dir, SyncDatabase *db, + DavResource *remote, LocalResource *res); int resource_pathlen_cmp(LocalResource *res1, LocalResource *res2, void *n); @@ -126,8 +133,7 @@ int sync_set_status(DavResource *res, char *status); int sync_remove_status(DavResource *res); UcxBuffer* sync_get_file_tag_data(SyncDirectory *dir, LocalResource *res); -UcxList* sync_get_file_tags(SyncDirectory *dir, LocalResource *res, DavBool *changed); -UcxList* sync_merge_tags(UcxList *tags1, UcxList *tags2); +UcxList* sync_get_file_tags(SyncDirectory *dir, LocalResource *res, DavBool *changed, char **newhash); int sync_tags_equal(UcxList *tags1, UcxList *tags2); int sync_store_tags(SyncDirectory *dir, const char *path, LocalResource *local, DavResource *res); int sync_store_tags_local(SyncDirectory *dir, LocalResource *local, const char *path, UcxList *tags); @@ -138,7 +144,16 @@ int *counter); int sync_mkdir(SyncDirectory *dir, DavResource *res, LocalResource *local); int sync_delete_remote_resource(SyncDirectory *dir, DavSession *sn, LocalResource *res, int *counter, UcxList **cols); -int sync_update_metadata(SyncDirectory *dir, DavSession *sn, DavResource *res, LocalResource *local); +MetadataHashes sync_set_metadata_properties( + SyncDirectory *dir, + DavSession *sn, + DavResource *res, + LocalResource *local); +int sync_update_metadata( + SyncDirectory *dir, + DavSession *sn, + DavResource *res, + LocalResource *local); void remove_deleted_conflicts(SyncDirectory *dir, SyncDatabase *db); @@ -152,7 +167,7 @@ int cmd_remove_tag(CmdArgs *args); int cmd_set_tags(CmdArgs *args); int cmd_list_tags(CmdArgs *args); -int cmd_tagopt(CmdArgs *args, int cmd); +int cmd_tagop(CmdArgs *args, int cmd); /* * gets the syncdir and resource path for a given file path
--- a/dav/tags.c Fri Mar 15 20:30:09 2019 +0100 +++ b/dav/tags.c Sun Mar 17 15:00:48 2019 +0100 @@ -398,6 +398,68 @@ return equal; } +char* create_tags_hash(UcxList *tags) { + if(!tags) { + return NULL; + } + UcxBuffer *buf = create_text_taglist(tags); + char *hash = dav_create_hash(buf->space, buf->size); + ucx_buffer_free(buf); + return hash; +} + +UcxList* merge_tags(UcxList *tags1, UcxList *tags2) { + // this map is used to check the existence of tags + UcxMap *tag_map = ucx_map_new(32); + // merged taglist + UcxList *new_tags = NULL; + + // add all local tags + UCX_FOREACH(elm, tags1) { + DavTag *t = elm->data; + ucx_map_cstr_put(tag_map, t->name, t); + DavTag *newt = calloc(1, sizeof(DavTag)); + newt->color = t->color ? strdup(t->color) : NULL; + newt->name = strdup(t->name); + new_tags = ucx_list_append(new_tags, newt); + } + // check if a remote tag is already in the map + // and if not add it to the new taglist + UCX_FOREACH(elm, tags2) { + DavTag *t = elm->data; + if(!ucx_map_cstr_get(tag_map, t->name)) { + DavTag *newt = calloc(1, sizeof(DavTag)); + newt->color = t->color ? strdup(t->color) : NULL; + newt->name = strdup(t->name); + new_tags = ucx_list_append(new_tags, newt); + } + } + + ucx_map_free(tag_map); + + return new_tags; +} + +void add_tag_colors(UcxList *taglist, UcxList *colored) { + UcxMap *tagmap = ucx_map_new(32); + UCX_FOREACH(elm, taglist) { + DavTag *tag = elm->data; + ucx_map_cstr_put(tagmap, tag->name, tag); + } + + UCX_FOREACH(elm, colored) { + DavTag *colored_tag = elm->data; + if(colored_tag->color) { + DavTag *tag = ucx_map_cstr_get(tagmap, colored_tag->name); + if(tag && !tag->color) { + tag->color = strdup(colored_tag->color); + } + } + } + + ucx_map_free(tagmap); +} + /* ----------- ----------- tag filter ---------------------- */ static size_t rtrimskip(scstr_t str, size_t skip) {
--- a/dav/tags.h Fri Mar 15 20:30:09 2019 +0100 +++ b/dav/tags.h Sun Mar 17 15:00:48 2019 +0100 @@ -88,7 +88,14 @@ UcxList* parse_macos_taglist(const char *buf, size_t length); UcxBuffer* create_macos_taglist(UcxList *tags); -int compare_taglists(UcxList *tags1, UcxList *tags2); +char* create_tags_hash(UcxList *tags); + +UcxList* merge_tags(UcxList *tags1, UcxList *tags2); + +/* + * Adds tag colors from the colored list to taglist if tags have the same name + */ +void add_tag_colors(UcxList *taglist, UcxList *colored); /* ----------- ----------- tag filter ---------------------- */
--- a/libidav/resource.c Fri Mar 15 20:30:09 2019 +0100 +++ b/libidav/resource.c Sun Mar 17 15:00:48 2019 +0100 @@ -907,6 +907,7 @@ int r = 0; if(ret == CURLE_OK && (status >= 200 && status < 300)) { res->session->error = DAV_OK; + res->exists = 0; // TODO: parse response // TODO: free res @@ -1012,6 +1013,7 @@ int status; if(!create_resource(res, &status)) { // resource successfully created + res->exists = 1; return 0; } @@ -1036,9 +1038,13 @@ long status = 0; curl_easy_getinfo (handle, CURLINFO_RESPONSE_CODE, &status); if(ret == CURLE_OK && (status >= 200 && status < 300)) { + res->exists = 1; return 1; } else { dav_session_set_error(sn, ret, status); + if(status == 404) { + res->exists = 0; + } return 0; } }
--- a/libidav/webdav.c Fri Mar 15 20:30:09 2019 +0100 +++ b/libidav/webdav.c Sun Mar 17 15:00:48 2019 +0100 @@ -297,6 +297,7 @@ //printf("response\n%s\n", rpbuf->space); // TODO: use PropfindParser resource = parse_propfind_response(sn, resource, rpbuf); + resource->exists = 1; sn->error = DAV_OK; } else { dav_session_set_error(sn, ret, status); @@ -338,6 +339,7 @@ dav_set_effective_href(sn, resource); resource = parse_propfind_response(sn, resource, rpbuf); sn->error = DAV_OK; + root->exists = 1; } else { dav_session_set_error(sn, ret, status); error = 1;
