changeset
adds semi functional download of splitted files
Thu, 28 Mar 2019 18:09:31 +0100
- author
- Olaf Wintermann <olaf.wintermann@gmail.com>
- date
- Thu, 28 Mar 2019 18:09:31 +0100
- changeset 544
- 9e85e1ec1155
- parent 543
- 2f85df8cd35e
- child 545
- 268157722a0e
adds semi functional download of splitted files
| dav/sync.c | file | annotate | diff | comparison | revisions | |
| dav/sync.h | file | annotate | diff | comparison | revisions | |
| libidav/utils.c | file | annotate | diff | comparison | revisions | |
| libidav/utils.h | file | annotate | diff | comparison | revisions |
--- a/dav/sync.c Thu Mar 28 14:18:54 2019 +0100 +++ b/dav/sync.c Thu Mar 28 18:09:31 2019 +0100 @@ -273,11 +273,20 @@ pthread_join(tid, &data); } +static char* create_local_path(SyncDirectory *dir, const char *path) { + char *local_path = util_concat_path(dir->path, path); + size_t local_path_len = strlen(local_path); + if(local_path[local_path_len-1] == '/') { + local_path[local_path_len-1] = '\0'; + } + return local_path; +} + static int res_matches_filter(SyncDirectory *dir, char *res_path) { // trash filter if (dir->trash) { sstr_t rpath = sstr(util_concat_path(dir->path, res_path)); - if (sstrprefix(rpath, sstr(dir->trash))) { + if (util_path_isrelated(dir->trash, rpath.ptr)) { free(rpath.ptr); return 1; } @@ -524,7 +533,7 @@ } int ret = 0; - DavResource *ls = dav_query(sn, "select D:getetag,idav:status,idav:tags,idav:finfo,idav:xattributes,`idav:content-hash` from / with depth = infinity"); + DavResource *ls = dav_query(sn, "select D:getetag,idav:status,idav:tags,idav:finfo,idav:xattributes,idav:split,`idav:content-hash` from / with depth = infinity"); if(!ls) { print_resource_error(sn, "/"); if(locked) { @@ -577,8 +586,6 @@ //UcxMap *svrres = ucx_map_new(db->resources->count); UcxMap *dbres = ucx_map_clone(db->resources, NULL, NULL); - UcxList *statls = NULL; - UcxList *stack = ucx_list_prepend(NULL, ls->children); while(stack) { DavResource *res = stack->data; @@ -652,7 +659,7 @@ // on the server ucx_map_cstr_remove(dbres, res->path); - if(res->children) { + if(!dav_get_property_ns(res, DAV_NS, "split") && res->children) { stack = ucx_list_prepend(stack, res->children); } res = res->next; @@ -676,7 +683,7 @@ // the first thing we need are all directories to put the files in UCX_FOREACH(elm, res_mkdir) { DavResource *res = elm->data; - char *local_path = util_concat_path(dir->path, res->path); + char *local_path = create_local_path(dir, res->path); if(sys_mkdir(local_path) && errno != EEXIST) { fprintf(stderr, "Cannot create directory %s: %s", @@ -701,8 +708,9 @@ break; } + DavBool issplit = dav_get_property_ns(res, DAV_NS, "split") ? 1 : 0; if(ucx_map_cstr_get(conflicts, res->path)) { - rename_conflict_file(dir, db, res->path); + rename_conflict_file(dir, db, res->path, issplit); } // download the resource @@ -721,7 +729,7 @@ LocalResource *local = ucx_map_cstr_get(db->resources, res->path); if(local) { printf("update: %s\n", res->path); - char *local_path = util_concat_path(dir->path, res->path); + char *local_path = create_local_path(dir, res->path); if(sync_store_metadata(dir, local_path, local, res)) { fprintf(stderr, "Metadata update failed: %s\n", res->path); sync_error++; @@ -823,12 +831,15 @@ } DavBool issplit = dav_get_property(res, "idav:split") ? TRUE : FALSE; + if(issplit) { + util_remove_trailing_pathseparator(res->path); + } RemoteChangeType type = cmd_getoption(a, "conflict") ? REMOTE_CHANGE_MODIFIED : REMOTE_CHANGE_CONFLICT_LOCAL_MODIFIED; LocalResource *local = ucx_map_cstr_get(db->resources, res->path); - char *local_path = util_concat_path(dir->path, res->path); + char *local_path = create_local_path(dir, res->path); SYS_STAT s; DavBool exists = 1; @@ -935,6 +946,106 @@ local->size = s->st_size; } +static UcxList* sync_download_changed_parts( + DavResource *res, + LocalResource *local, + FILE *out, + size_t blocksize, + uint64_t *blockcount, + int64_t *truncate_file, + int *err) +{ + UcxList *updates = NULL; + + size_t local_numparts = local ? local->numparts : 0; + + int error = 0; + + UcxBuffer *buf = ucx_buffer_new(NULL, blocksize, 0); + + int64_t maxsize = -1; + + DavResource *part = res->children; + uint64_t i = 0; + while(part) { + char *res_name = part->name; + while(res_name[0] == '0' && res_name[1] != '\0') { + res_name++; + } + uint64_t partnum = 0; + if(util_strtouint(res_name, &partnum)) { + DavBool download_part = FALSE; + char *etag = dav_get_string_property_ns(part, "DAV:", "getetag"); + if(partnum >= local_numparts) { + // new part + download_part = TRUE; + } else { + FilePart p = local->parts[partnum]; // local is always non-null here + if(etag) { + if(nullstrcmp(etag, p.etag)) { + download_part = TRUE; + } + } + } + + uint64_t offset; + int mul_err = util_uint_mul(partnum, blocksize, &offset); + if(mul_err || offset >= INT64_MAX) { + error = 1; + fprintf(stderr, "Error: part number too high\n"); + break; + } + + if(download_part) { + if(fseeko(out, offset, SEEK_SET)) { + error = 1; + fprintf(stderr, "Error: fseek failed: %s\n", strerror(errno)); + break; + } + buf->pos = 0; + buf->size = 0; + if(dav_get_content(part, buf,(dav_write_func)ucx_buffer_write)) { + fprintf(stderr, "Error: cannot download part: %s\n", part->name); + error = 1; + break; + } + printf("offset: %lu length: %lu\n", offset, (uint64_t)buf->size); + if(fwrite(buf->space, 1, buf->size, out) == 0) { + perror("write"); + error = 1; + break; + } + + FilePart *update = calloc(1, sizeof(FilePart)); + update->block = partnum; + update->etag = etag ? strdup(etag) : NULL; + update->hash = dav_create_hash(buf->space, buf->size); + updates = ucx_list_append(updates, update); + } + + + + i++; + } // else: res is not a regular file part + part = part->next; + } + + ucx_buffer_free(buf); + + if(error) { + *err = 1; + ucx_list_free_content(updates, (ucx_destructor)filepart_free); + return NULL; + } + + *err = 0; + *blockcount = i; + *truncate_file = maxsize; + return updates; +} + + + int sync_get_resource( CmdArgs *a, SyncDirectory *dir, @@ -944,49 +1055,87 @@ int *counter) { LocalResource *local = ucx_map_cstr_get(db->resources, path); - char *local_path = util_concat_path(dir->path, path); + char *local_path = create_local_path(dir, path); char *etag = dav_get_string_property(res, "D:getetag"); SYS_STAT s; memset(&s, 0, sizeof(SYS_STAT)); + + char *blocksize_str = dav_get_string_property_ns(res, DAV_NS, "split"); + int64_t blocksize = 0; + DavBool issplit = FALSE; + if(blocksize_str) { + if(!util_strtouint(blocksize_str, &blocksize)) { + fprintf(stderr, "Error: split property does not contain an integer.\n"); + return 1; + } + issplit = TRUE; + } + UcxList *part_updates = NULL; + uint64_t blockcount = 0; + + if(res->iscollection && !issplit) { + // why are we here? + return 0; + } int ret = 0; - char *tmp_path = create_tmp_download_path(local_path); - - if(!tmp_path) { - fprintf(stderr, "Cannot create tmp path for %s\n", local_path); - free(local_path); - return -1; - } - FILE *out = sys_fopen(tmp_path, "wb"); + + char *tmp_path = NULL; + FILE *out = NULL; + if(!issplit) { + tmp_path = create_tmp_download_path(local_path); + if(!tmp_path) { + fprintf(stderr, "Cannot create tmp path for %s\n", local_path); + free(local_path); + return -1; + } + out = sys_fopen(tmp_path , "wb"); + } else { + out = sys_fopen(local_path, "r+b"); + if(!out && errno == ENOENT) { + out = sys_fopen(local_path, "wb"); + } + } if(!out) { fprintf(stderr, "Cannot open output file: %s\n", local_path); free(local_path); - free(tmp_path); + if(tmp_path) { + free(tmp_path); + } return -1; } + + int64_t truncate_file = -1; printf("get: %s\n", path); - if(dav_get_content(res, out, (dav_write_func)fwrite)) { - ret = -1; + if(issplit) { + part_updates = sync_download_changed_parts(res, local, out, blocksize, &blockcount, &truncate_file, &ret); + } else { + ret = dav_get_content(res, out, (dav_write_func)fwrite); } fclose(out); + if(truncate_file >= 0) { + truncate(tmp_path, truncate_file); + } if(ret == 0) { (*counter)++; - - if(dir->trash && dir->backuppull) { - move_to_trash(dir, local_path); - } - if(sys_rename(tmp_path, local_path)) { - fprintf( - stderr, - "Cannot rename file %s to %s\n", - tmp_path, - local_path); - perror(""); - free(tmp_path); - free(local_path); - return -1; + + if(tmp_path) { + if(dir->trash && dir->backuppull) { + move_to_trash(dir, local_path); + } + if(sys_rename(tmp_path, local_path)) { + fprintf( + stderr, + "Cannot rename file %s to %s\n", + tmp_path, + local_path); + perror(""); + free(tmp_path); + free(local_path); + return -1; + } } if(sys_stat(local_path, &s)) { @@ -1009,23 +1158,27 @@ free(local->etag); } + update_parts(local, part_updates, blockcount); + // set metadata from stat local->etag = strdup(etag); sync_set_metadata_from_stat(local, &s); local->skipped = FALSE; - } else { + } else if(tmp_path) { if(sys_unlink(tmp_path)) { fprintf(stderr, "Cannot remove tmp file: %s\n", tmp_path); } } - free(tmp_path); + if(tmp_path) { + free(tmp_path); + } free(local_path); return ret; } int sync_remove_local_resource(SyncDirectory *dir, LocalResource *res) { - char *local_path = util_concat_path(dir->path, res->path); + char *local_path = create_local_path(dir, res->path); SYS_STAT s; if(sys_stat(local_path, &s)) { free(local_path); @@ -1057,7 +1210,7 @@ int sync_remove_local_directory(SyncDirectory *dir, LocalResource *res) { int ret = 0; - char *local_path = util_concat_path(dir->path, res->path); + char *local_path = create_local_path(dir, res->path); printf("delete: %s\n", res->path); if(rmdir(local_path)) { @@ -1069,10 +1222,30 @@ return ret; } -void rename_conflict_file(SyncDirectory *dir, SyncDatabase *db, char *path) { - char *local_path = util_concat_path(dir->path, path); +int copy_file(const char *from, const char *to) { + FILE *in = sys_fopen(from, "rb"); + if(!in) { + return 1; + } + FILE *out = sys_fopen(to, "wb"); + if(!out) { + fclose(in); + return 1; + } + + ucx_stream_copy(in, out, (read_func)fread, (write_func)fwrite); + fclose(in); + fclose(out); +} + +typedef int (*renamefunc)(const char*,const char*); + +void rename_conflict_file(SyncDirectory *dir, SyncDatabase *db, char *path, DavBool copy) { + char *local_path = create_local_path(dir, path); char *parent = util_parent_path(local_path); + renamefunc fn = copy ? copy_file : sys_rename; + int rev = 0; SYS_STAT s; int loop = 1; @@ -1096,7 +1269,7 @@ if(errno == ENOENT) { loop = 0; printf("conflict: %s\n", local_path); - if(sys_rename(local_path, new_path.ptr)) { + if(fn(local_path, new_path.ptr)) { //printf("errno: %d\n", errno); fprintf( stderr, @@ -1637,7 +1810,7 @@ } } - char *file_path = util_concat_path(dir->path, resource->path); + char *file_path = create_local_path(dir, resource->path); SYS_STAT s; if(sys_stat(file_path, &s)) { if(errno == ENOENT) { @@ -1741,7 +1914,7 @@ // download the resource if(!sync_shutdown) { if(resource->isdirectory) { - char *local_path = util_concat_path(dir->path, res->path); + char *local_path = create_local_path(dir, res->path); if(sys_mkdir(local_path) && errno != EEXIST) { fprintf(stderr, "Cannot create directory %s: %s", @@ -1803,7 +1976,7 @@ char *p = stack->data; stack = ucx_list_remove(stack, stack); - char *local_path = util_concat_path(dir->path, p); + char *local_path = create_local_path(dir, p); SYS_DIR local_dir = sys_opendir(local_path); if(!local_dir) { @@ -1885,7 +2058,7 @@ } LocalResource* local_resource_new(SyncDirectory *dir, SyncDatabase *db, char *path, int *isdir) { - char *file_path = util_concat_path(dir->path, path); + char *file_path = create_local_path(dir, path); SYS_STAT s; if(sys_stat(file_path, &s)) { fprintf(stderr, "Cannot stat file %s\n", file_path); @@ -2011,7 +2184,7 @@ } if((dir->metadata & FINFO_XATTR) == FINFO_XATTR) { - char *path = util_concat_path(dir->path, db_res->path); + char *path = create_local_path(dir, db_res->path); XAttributes *xattr = file_get_attributes(path); // test if xattr are added, removed or changed if((db_res->xattr_hash && !xattr) || @@ -2448,7 +2621,7 @@ UcxBuffer *buf = NULL; if(dir->tagconfig->store == TAG_STORE_XATTR) { ssize_t tag_length = 0; - char *local_path = util_concat_path(dir->path, res->path); + char *local_path = create_local_path(dir, res->path); char* tag_data = xattr_get( local_path, dir->tagconfig->xattr_name, @@ -2886,7 +3059,7 @@ return updated_parts; } -static void update_parts(LocalResource *local, UcxList *updates, uint64_t numparts) { +void update_parts(LocalResource *local, UcxList *updates, uint64_t numparts) { size_t old_num = local->numparts; if(old_num > numparts) { // free old parts @@ -2938,7 +3111,7 @@ LocalResource *local, int *counter) { - char *local_path = util_concat_path(dir->path, res->path); + char *local_path = create_local_path(dir, res->path); SYS_STAT s; if(sys_stat(local_path, &s)) { @@ -3265,7 +3438,7 @@ UcxMapIterator i = ucx_map_iterator(db->conflict); LocalResource *res; UCX_MAP_FOREACH(key, res, i) { - char *path = util_concat_path(dir->path, res->path); + char *path = create_local_path(dir, res->path); SYS_STAT s; if(sys_stat(path, &s)) { if(errno == ENOENT) { @@ -3386,7 +3559,7 @@ LocalResource *res; UCX_MAP_FOREACH(key, res, i) { printf("delete: %s\n", res->path); - char *path = util_concat_path(dir->path, res->path); + char *path = create_local_path(dir, res->path); if(sys_unlink(path)) { if(errno != ENOENT) { perror("unlink");
--- a/dav/sync.h Thu Mar 28 14:18:54 2019 +0100 +++ b/dav/sync.h Thu Mar 28 18:09:31 2019 +0100 @@ -73,7 +73,7 @@ REMOTE_CHANGE_DELETED, REMOTE_CHANGE_CONFLICT_LOCAL_MODIFIED, REMOTE_CHANGE_METADATA, - REMOTE_CHANGE_MKDIR + REMOTE_CHANGE_MKDIR, }; typedef enum RemoteChangeType RemoteChangeType; @@ -81,7 +81,7 @@ DavResource *resource; RemoteChangeType type; } RemoteChange; - + void print_usage(char *cmd); pthread_t start_sighandler(pthread_mutex_t *mutex) ; @@ -109,7 +109,7 @@ int *counter); int sync_remove_local_resource(SyncDirectory *dir, LocalResource *res); int sync_remove_local_directory(SyncDirectory *dir, LocalResource *res); -void rename_conflict_file(SyncDirectory *dir, SyncDatabase *db, char *path); +void rename_conflict_file(SyncDirectory *dir, SyncDatabase *db, char *path, DavBool copy); char* create_tmp_download_path(char *path); void move_to_trash(SyncDirectory *dir, char *path); @@ -165,6 +165,8 @@ DavResource *res, LocalResource *local); +void update_parts(LocalResource *local, UcxList *updates, uint64_t numparts); + void remove_deleted_conflicts(SyncDirectory *dir, SyncDatabase *db); int cmd_resolve_conflicts(CmdArgs *args);
--- a/libidav/utils.c Thu Mar 28 14:18:54 2019 +0100 +++ b/libidav/utils.c Thu Mar 28 18:09:31 2019 +0100 @@ -863,3 +863,14 @@ ucx_buffer_free(buf); return str; } + +void util_remove_trailing_pathseparator(char *path) { + size_t len = strlen(path); + if(len < 2) { + return; + } + + if(path[len-1] == '/') { + path[len-1] = '\0'; + } +}
