dav: comparison dav/sync.c

-:2f85df8cd35e
+:9e85e1ec1155
 pthread_mutex_unlock(mutex);
 void *data;
 pthread_join(tid, &data);
 }
+static char* create_local_path(SyncDirectory *dir, const char *path) {
+char *local_path = util_concat_path(dir->path, path);
+size_t local_path_len = strlen(local_path);
+if(local_path[local_path_len-1] == '/') {
+local_path[local_path_len-1] = '\0';
+}
+return local_path;
+}
 static int res_matches_filter(SyncDirectory *dir, char *res_path) {
 // trash filter
 if (dir->trash) {
 sstr_t rpath = sstr(util_concat_path(dir->path, res_path));
-if (sstrprefix(rpath, sstr(dir->trash))) {
+if (util_path_isrelated(dir->trash, rpath.ptr)) {
 free(rpath.ptr);
 return 1;
 }
 free(rpath.ptr);
 }
 locked = TRUE;
 locktokenfile = create_locktoken_file(dir->name, lock->token);
 }
 int ret = 0;
-DavResource *ls = dav_query(sn, "select D:getetag,idav:status,idav:tags,idav:finfo,idav:xattributes,`idav:content-hash` from / with depth = infinity");
+DavResource *ls = dav_query(sn, "select D:getetag,idav:status,idav:tags,idav:finfo,idav:xattributes,idav:split,`idav:content-hash` from / with depth = infinity");
 if(!ls) {
 print_resource_error(sn, "/");
 if(locked) {
 if(dav_unlock(root)) {
 print_resource_error(sn, "/");
 UcxList *res_broken = NULL;
 UcxList *lres_removed = NULL; // list of LocalResource*
 //UcxMap *svrres = ucx_map_new(db->resources->count);
 UcxMap *dbres = ucx_map_clone(db->resources, NULL, NULL);
-UcxList *statls = NULL;
 UcxList *stack = ucx_list_prepend(NULL, ls->children);
 while(stack) {
 DavResource *res = stack->data;
 stack = ucx_list_remove(stack, stack);
 // remove every server resource from dbres
 // all remaining elements are the resources that are removed
 // on the server
 ucx_map_cstr_remove(dbres, res->path);
-if(res->children) {
+if(!dav_get_property_ns(res, DAV_NS, "split") && res->children) {
 stack = ucx_list_prepend(stack, res->children);
 }
 res = res->next;
 }
 }
 }
 // the first thing we need are all directories to put the files in
 UCX_FOREACH(elm, res_mkdir) {
 DavResource *res = elm->data;
-char *local_path = util_concat_path(dir->path, res->path);
+char *local_path = create_local_path(dir, res->path);
 if(sys_mkdir(local_path) && errno != EEXIST) {
 fprintf(stderr,
 "Cannot create directory %s: %s",
 local_path, strerror(errno));
 }
 DavResource *res = elm->data;
 if(sync_shutdown) {
 break;
 }
+DavBool issplit = dav_get_property_ns(res, DAV_NS, "split") ? 1 : 0;
 if(ucx_map_cstr_get(conflicts, res->path)) {
-rename_conflict_file(dir, db, res->path);
+rename_conflict_file(dir, db, res->path, issplit);
 }
 // download the resource
 if(sync_get_resource(a, dir, res->path, res, db, &sync_success)) {
 fprintf(stderr, "resource download failed: %s\n", res->path);
 }
 LocalResource *local = ucx_map_cstr_get(db->resources, res->path);
 if(local) {
 printf("update: %s\n", res->path);
-char *local_path = util_concat_path(dir->path, res->path);
+char *local_path = create_local_path(dir, res->path);
 if(sync_store_metadata(dir, local_path, local, res)) {
 fprintf(stderr, "Metadata update failed: %s\n", res->path);
 sync_error++;
 } else {
 struct stat s;
 fprintf(stderr, "Error: resource %s has no etag\n", res->path);
 return REMOTE_NO_CHANGE;
 }
 DavBool issplit = dav_get_property(res, "idav:split") ? TRUE : FALSE;
+if(issplit) {
+util_remove_trailing_pathseparator(res->path);
+}
 RemoteChangeType type = cmd_getoption(a, "conflict") ?
 REMOTE_CHANGE_MODIFIED : REMOTE_CHANGE_CONFLICT_LOCAL_MODIFIED;
 LocalResource *local = ucx_map_cstr_get(db->resources, res->path);
-char *local_path = util_concat_path(dir->path, res->path);
+char *local_path = create_local_path(dir, res->path);
 SYS_STAT s;
 DavBool exists = 1;
 if(sys_stat(local_path, &s)) {
 if(errno != ENOENT) {
 local->mode = s->st_mode & 07777;
 local->uid = s->st_uid;
 local->gid = s->st_gid;
 local->size = s->st_size;
 }
+static UcxList* sync_download_changed_parts(
+DavResource *res,
+LocalResource *local,
+FILE *out,
+size_t blocksize,
+uint64_t *blockcount,
+int64_t *truncate_file,
+int *err)
+{
+UcxList *updates = NULL;
+size_t local_numparts = local ? local->numparts : 0;
+int error = 0;
+UcxBuffer *buf = ucx_buffer_new(NULL, blocksize, 0);
+int64_t maxsize = -1;
+DavResource *part = res->children;
+uint64_t i = 0;
+while(part) {
+char *res_name = part->name;
+while(res_name[0] == '0' && res_name[1] != '\0') {
+res_name++;
+}
+uint64_t partnum = 0;
+if(util_strtouint(res_name, &partnum)) {
+DavBool download_part = FALSE;
+char *etag = dav_get_string_property_ns(part, "DAV:", "getetag");
+if(partnum >= local_numparts) {
+// new part
+download_part = TRUE;
+} else {
+FilePart p = local->parts[partnum]; // local is always non-null here
+if(etag) {
+if(nullstrcmp(etag, p.etag)) {
+download_part = TRUE;
+}
+}
+}
+uint64_t offset;
+int mul_err = util_uint_mul(partnum, blocksize, &offset);
+if(mul_err || offset >= INT64_MAX) {
+error = 1;
+fprintf(stderr, "Error: part number too high\n");
+break;
+}
+if(download_part) {
+if(fseeko(out, offset, SEEK_SET)) {
+error = 1;
+fprintf(stderr, "Error: fseek failed: %s\n", strerror(errno));
+break;
+}
+buf->pos = 0;
+buf->size = 0;
+if(dav_get_content(part, buf,(dav_write_func)ucx_buffer_write)) {
+fprintf(stderr, "Error: cannot download part: %s\n", part->name);
+error = 1;
+break;
+}
+printf("offset: %lu length: %lu\n", offset, (uint64_t)buf->size);
+if(fwrite(buf->space, 1, buf->size, out) == 0) {
+perror("write");
+error = 1;
+break;
+}
+FilePart *update = calloc(1, sizeof(FilePart));
+update->block = partnum;
+update->etag = etag ? strdup(etag) : NULL;
+update->hash = dav_create_hash(buf->space, buf->size);
+updates = ucx_list_append(updates, update);
+}
+i++;
+} // else: res is not a regular file part
+part = part->next;
+}
+ucx_buffer_free(buf);
+if(error) {
+*err = 1;
+ucx_list_free_content(updates, (ucx_destructor)filepart_free);
+return NULL;
+}
+*err = 0;
+*blockcount = i;
+*truncate_file = maxsize;
+return updates;
+}
 int sync_get_resource(
 CmdArgs *a,
 SyncDirectory *dir,
 const char *path,
 DavResource *res,
 SyncDatabase *db,
 int *counter)
 {
 LocalResource *local = ucx_map_cstr_get(db->resources, path);
-char *local_path = util_concat_path(dir->path, path);
+char *local_path = create_local_path(dir, path);
 char *etag = dav_get_string_property(res, "D:getetag");
 SYS_STAT s;
 memset(&s, 0, sizeof(SYS_STAT));
+char *blocksize_str = dav_get_string_property_ns(res, DAV_NS, "split");
+int64_t blocksize = 0;
+DavBool issplit = FALSE;
+if(blocksize_str) {
+if(!util_strtouint(blocksize_str, &blocksize)) {
+fprintf(stderr, "Error: split property does not contain an integer.\n");
+return 1;
+}
+issplit = TRUE;
+}
+UcxList *part_updates = NULL;
+uint64_t blockcount = 0;
+if(res->iscollection && !issplit) {
+// why are we here?
+return 0;
+}
 int ret = 0;
-char *tmp_path = create_tmp_download_path(local_path);
+char *tmp_path = NULL;
-if(!tmp_path) {
+FILE *out = NULL;
-fprintf(stderr, "Cannot create tmp path for %s\n", local_path);
+if(!issplit) {
-free(local_path);
+tmp_path = create_tmp_download_path(local_path);
-return -1;
+if(!tmp_path) {
-}
+fprintf(stderr, "Cannot create tmp path for %s\n", local_path);
-FILE *out = sys_fopen(tmp_path, "wb");
+free(local_path);
+return -1;
+}
+out = sys_fopen(tmp_path , "wb");
+} else {
+out = sys_fopen(local_path, "r+b");
+if(!out && errno == ENOENT) {
+out = sys_fopen(local_path, "wb");
+}
+}
 if(!out) {
 fprintf(stderr, "Cannot open output file: %s\n", local_path);
 free(local_path);
-free(tmp_path);
+if(tmp_path) {
-return -1;
+free(tmp_path);
 }
+return -1;
+}
+int64_t truncate_file = -1;
 printf("get: %s\n", path);
-if(dav_get_content(res, out, (dav_write_func)fwrite)) {
+if(issplit) {
-ret = -1;
+part_updates = sync_download_changed_parts(res, local, out, blocksize, &blockcount, &truncate_file, &ret);
+} else {
+ret = dav_get_content(res, out, (dav_write_func)fwrite);
 }
 fclose(out);
+if(truncate_file >= 0) {
+truncate(tmp_path, truncate_file);
+}
 if(ret == 0) {
 (*counter)++;
-if(dir->trash && dir->backuppull) {
+if(tmp_path) {
-move_to_trash(dir, local_path);
+if(dir->trash && dir->backuppull) {
-}
+move_to_trash(dir, local_path);
-if(sys_rename(tmp_path, local_path)) {
+}
-fprintf(
+if(sys_rename(tmp_path, local_path)) {
-stderr,
+fprintf(
-"Cannot rename file %s to %s\n",
+stderr,
-tmp_path,
+"Cannot rename file %s to %s\n",
-local_path);
+tmp_path,
-perror("");
+local_path);
-free(tmp_path);
+perror("");
-free(local_path);
+free(tmp_path);
-return -1;
+free(local_path);
+return -1;
+}
 }
 if(sys_stat(local_path, &s)) {
 fprintf(stderr,
 "Cannot stat file %s: %s\n", local_path, strerror(errno));
 if(local->etag) {
 free(local->etag);
 }
+update_parts(local, part_updates, blockcount);
 // set metadata from stat
 local->etag = strdup(etag);
 sync_set_metadata_from_stat(local, &s);
 local->skipped = FALSE;
-} else {
+} else if(tmp_path) {
 if(sys_unlink(tmp_path)) {
 fprintf(stderr, "Cannot remove tmp file: %s\n", tmp_path);
 }
 }
-free(tmp_path);
+if(tmp_path) {
+free(tmp_path);
+}
 free(local_path);
 return ret;
 }
 int sync_remove_local_resource(SyncDirectory *dir, LocalResource *res) {
-char *local_path = util_concat_path(dir->path, res->path);
+char *local_path = create_local_path(dir, res->path);
 SYS_STAT s;
 if(sys_stat(local_path, &s)) {
 free(local_path);
 return -2;
 }
 return ret;
 }
 int sync_remove_local_directory(SyncDirectory *dir, LocalResource *res) {
 int ret = 0;
-char *local_path = util_concat_path(dir->path, res->path);
+char *local_path = create_local_path(dir, res->path);
 printf("delete: %s\n", res->path);
 if(rmdir(local_path)) {
 fprintf(stderr, "rmdir: %s : %s", local_path, strerror(errno));
 ret = 1;
 free(local_path);
 return ret;
 }
-void rename_conflict_file(SyncDirectory *dir, SyncDatabase *db, char *path) {
+int copy_file(const char *from, const char *to) {
-char *local_path = util_concat_path(dir->path, path);
+FILE *in = sys_fopen(from, "rb");
+if(!in) {
+return 1;
+}
+FILE *out = sys_fopen(to, "wb");
+if(!out) {
+fclose(in);
+return 1;
+}
+ucx_stream_copy(in, out, (read_func)fread, (write_func)fwrite);
+fclose(in);
+fclose(out);
+}
+typedef int (*renamefunc)(const char*,const char*);
+void rename_conflict_file(SyncDirectory *dir, SyncDatabase *db, char *path, DavBool copy) {
+char *local_path = create_local_path(dir, path);
 char *parent = util_parent_path(local_path);
+renamefunc fn = copy ? copy_file : sys_rename;
 int rev = 0;
 SYS_STAT s;
 int loop = 1;
 do {
 if(sys_stat(new_path.ptr, &s)) {
 if(errno == ENOENT) {
 loop = 0;
 printf("conflict: %s\n", local_path);
-if(sys_rename(local_path, new_path.ptr)) {
+if(fn(local_path, new_path.ptr)) {
 //printf("errno: %d\n", errno);
 fprintf(
 stderr,
 "Cannot rename file %s to %s\n",
 local_path,
 if(!resource) {
 continue;
 }
 }
-char *file_path = util_concat_path(dir->path, resource->path);
+char *file_path = create_local_path(dir, resource->path);
 SYS_STAT s;
 if(sys_stat(file_path, &s)) {
 if(errno == ENOENT) {
 if(restore_removed) {
 deleted = ucx_list_prepend(deleted, resource);
 }
 // download the resource
 if(!sync_shutdown) {
 if(resource->isdirectory) {
-char *local_path = util_concat_path(dir->path, res->path);
+char *local_path = create_local_path(dir, res->path);
 if(sys_mkdir(local_path) && errno != EEXIST) {
 fprintf(stderr,
 "Cannot create directory %s: %s",
 local_path, strerror(errno));
 }
 // get a directory path from the stack and read all entries
 // if an entry is a directory, put it on the stack
 char *p = stack->data;
 stack = ucx_list_remove(stack, stack);
-char *local_path = util_concat_path(dir->path, p);
+char *local_path = create_local_path(dir, p);
 SYS_DIR local_dir = sys_opendir(local_path);
 if(!local_dir) {
 fprintf(stderr, "Cannot open directory %s\n", local_path);
 } else {
 return resources;
 }
 LocalResource* local_resource_new(SyncDirectory *dir, SyncDatabase *db, char *path, int *isdir) {
-char *file_path = util_concat_path(dir->path, path);
+char *file_path = create_local_path(dir, path);
 SYS_STAT s;
 if(sys_stat(file_path, &s)) {
 fprintf(stderr, "Cannot stat file %s\n", file_path);
 free(file_path);
 return NULL;
 res->metadata_updated = 1;
 }
 }
 if((dir->metadata & FINFO_XATTR) == FINFO_XATTR) {
-char *path = util_concat_path(dir->path, db_res->path);
+char *path = create_local_path(dir, db_res->path);
 XAttributes *xattr = file_get_attributes(path);
 // test if xattr are added, removed or changed
 if((db_res->xattr_hash && !xattr) ||
 (!db_res->xattr_hash && xattr) ||
 (xattr && db_res->xattr_hash && strcmp(xattr->hash, db_res->xattr_hash)))
 return res->cached_tags;
 }
 UcxBuffer *buf = NULL;
 if(dir->tagconfig->store == TAG_STORE_XATTR) {
 ssize_t tag_length = 0;
-char *local_path = util_concat_path(dir->path, res->path);
+char *local_path = create_local_path(dir, res->path);
 char* tag_data = xattr_get(
 local_path,
 dir->tagconfig->xattr_name,
 &tag_length);
 free(local_path);
 *err = 0;
 return updated_parts;
 }
-static void update_parts(LocalResource *local, UcxList *updates, uint64_t numparts) {
+void update_parts(LocalResource *local, UcxList *updates, uint64_t numparts) {
 size_t old_num = local->numparts;
 if(old_num > numparts) {
 // free old parts
 for(size_t i=numparts;i<old_num;i++) {
 FilePart p = local->parts[i];
 SyncDirectory *dir,
 DavResource *res,
 LocalResource *local,
 int *counter)
 {
-char *local_path = util_concat_path(dir->path, res->path);
+char *local_path = create_local_path(dir, res->path);
 SYS_STAT s;
 if(sys_stat(local_path, &s)) {
 fprintf(stderr, "Cannot stat file: %s\n", local_path);
 perror("");
 int numdc = 0;
 UcxMapIterator i = ucx_map_iterator(db->conflict);
 LocalResource *res;
 UCX_MAP_FOREACH(key, res, i) {
-char *path = util_concat_path(dir->path, res->path);
+char *path = create_local_path(dir, res->path);
 SYS_STAT s;
 if(sys_stat(path, &s)) {
 if(errno == ENOENT) {
 dc[numdc] = res->path;
 numdc++;
 // delete all conflict files
 UcxMapIterator i = ucx_map_iterator(db->conflict);
 LocalResource *res;
 UCX_MAP_FOREACH(key, res, i) {
 printf("delete: %s\n", res->path);
-char *path = util_concat_path(dir->path, res->path);
+char *path = create_local_path(dir, res->path);
 if(sys_unlink(path)) {
 if(errno != ENOENT) {
 perror("unlink");
 num_err++;
 }

Mercurial > hg > dav / file comparison

comparison: dav/sync.c

dav/sync.c