adds semi functional download of splitted files

Thu, 28 Mar 2019 18:09:31 +0100

author
Olaf Wintermann <olaf.wintermann@gmail.com>
date
Thu, 28 Mar 2019 18:09:31 +0100
changeset 544
9e85e1ec1155
parent 543
2f85df8cd35e
child 545
268157722a0e

adds semi functional download of splitted files

dav/sync.c file | annotate | diff | comparison | revisions
dav/sync.h file | annotate | diff | comparison | revisions
libidav/utils.c file | annotate | diff | comparison | revisions
libidav/utils.h file | annotate | diff | comparison | revisions
--- a/dav/sync.c	Thu Mar 28 14:18:54 2019 +0100
+++ b/dav/sync.c	Thu Mar 28 18:09:31 2019 +0100
@@ -273,11 +273,20 @@
     pthread_join(tid, &data);
 }
 
+static char* create_local_path(SyncDirectory *dir, const char *path) {
+    char *local_path = util_concat_path(dir->path, path);
+    size_t local_path_len = strlen(local_path);
+    if(local_path[local_path_len-1] == '/') {
+        local_path[local_path_len-1] = '\0';
+    }
+    return local_path;
+}
+
 static int res_matches_filter(SyncDirectory *dir, char *res_path) {
     // trash filter
     if (dir->trash) {
         sstr_t rpath = sstr(util_concat_path(dir->path, res_path));
-        if (sstrprefix(rpath, sstr(dir->trash))) {
+        if (util_path_isrelated(dir->trash, rpath.ptr)) {
             free(rpath.ptr);
             return 1;
         }
@@ -524,7 +533,7 @@
     }
     
     int ret = 0;
-    DavResource *ls = dav_query(sn, "select D:getetag,idav:status,idav:tags,idav:finfo,idav:xattributes,`idav:content-hash` from / with depth = infinity");
+    DavResource *ls = dav_query(sn, "select D:getetag,idav:status,idav:tags,idav:finfo,idav:xattributes,idav:split,`idav:content-hash` from / with depth = infinity");
     if(!ls) {
         print_resource_error(sn, "/");
         if(locked) {
@@ -577,8 +586,6 @@
     //UcxMap *svrres = ucx_map_new(db->resources->count);
     UcxMap *dbres = ucx_map_clone(db->resources, NULL, NULL);
     
-    UcxList *statls = NULL;
-    
     UcxList *stack = ucx_list_prepend(NULL, ls->children);
     while(stack) {
         DavResource *res = stack->data;
@@ -652,7 +659,7 @@
             // on the server
             ucx_map_cstr_remove(dbres, res->path);
             
-            if(res->children) {
+            if(!dav_get_property_ns(res, DAV_NS, "split") && res->children) {
                 stack = ucx_list_prepend(stack, res->children);
             }
             res = res->next;
@@ -676,7 +683,7 @@
     // the first thing we need are all directories to put the files in
     UCX_FOREACH(elm, res_mkdir) {
         DavResource *res = elm->data;
-        char *local_path = util_concat_path(dir->path, res->path);
+        char *local_path = create_local_path(dir, res->path);
         if(sys_mkdir(local_path) && errno != EEXIST) {
             fprintf(stderr,
                     "Cannot create directory %s: %s",
@@ -701,8 +708,9 @@
             break;
         }
         
+        DavBool issplit = dav_get_property_ns(res, DAV_NS, "split") ? 1 : 0;  
         if(ucx_map_cstr_get(conflicts, res->path)) {
-            rename_conflict_file(dir, db, res->path);
+            rename_conflict_file(dir, db, res->path, issplit);
         }
         
         // download the resource
@@ -721,7 +729,7 @@
         LocalResource *local = ucx_map_cstr_get(db->resources, res->path);
         if(local) {
             printf("update: %s\n", res->path);
-            char *local_path = util_concat_path(dir->path, res->path);
+            char *local_path = create_local_path(dir, res->path);
             if(sync_store_metadata(dir, local_path, local, res)) {
                 fprintf(stderr, "Metadata update failed: %s\n", res->path);
                 sync_error++;
@@ -823,12 +831,15 @@
     }
     
     DavBool issplit = dav_get_property(res, "idav:split") ? TRUE : FALSE;
+    if(issplit) {
+        util_remove_trailing_pathseparator(res->path);
+    }
     
     RemoteChangeType type = cmd_getoption(a, "conflict") ?
             REMOTE_CHANGE_MODIFIED : REMOTE_CHANGE_CONFLICT_LOCAL_MODIFIED;
     
     LocalResource *local = ucx_map_cstr_get(db->resources, res->path);
-    char *local_path = util_concat_path(dir->path, res->path);
+    char *local_path = create_local_path(dir, res->path);
     
     SYS_STAT s;
     DavBool exists = 1;
@@ -935,6 +946,106 @@
     local->size = s->st_size;
 }
 
+static UcxList* sync_download_changed_parts(
+        DavResource *res,
+        LocalResource *local,
+        FILE *out,
+        size_t blocksize,
+        uint64_t *blockcount,
+        int64_t *truncate_file,
+        int *err)
+{
+    UcxList *updates = NULL;
+    
+    size_t local_numparts = local ? local->numparts : 0;
+    
+    int error = 0;
+    
+    UcxBuffer *buf = ucx_buffer_new(NULL, blocksize, 0);
+    
+    int64_t maxsize = -1;
+    
+    DavResource *part = res->children;
+    uint64_t i = 0;
+    while(part) {
+        char *res_name = part->name;
+        while(res_name[0] == '0' && res_name[1] != '\0') {
+            res_name++;
+        }
+        uint64_t partnum = 0;
+        if(util_strtouint(res_name, &partnum)) {
+            DavBool download_part = FALSE;
+            char *etag = dav_get_string_property_ns(part, "DAV:", "getetag");
+            if(partnum >= local_numparts) {
+                // new part
+                download_part = TRUE;
+            } else {
+                FilePart p = local->parts[partnum]; // local is always non-null here
+                if(etag) {
+                    if(nullstrcmp(etag, p.etag)) {
+                        download_part = TRUE;
+                    }
+                }
+            }
+            
+            uint64_t offset;
+            int mul_err = util_uint_mul(partnum, blocksize, &offset);
+            if(mul_err || offset >= INT64_MAX) {
+                error = 1;
+                fprintf(stderr, "Error: part number too high\n");
+                break;
+            }
+            
+            if(download_part) {
+                if(fseeko(out, offset, SEEK_SET)) {
+                    error = 1;
+                    fprintf(stderr, "Error: fseek failed: %s\n", strerror(errno));
+                    break;
+                }
+                buf->pos = 0;
+                buf->size = 0;
+                if(dav_get_content(part, buf,(dav_write_func)ucx_buffer_write)) {
+                    fprintf(stderr, "Error: cannot download part: %s\n", part->name);
+                    error = 1;
+                    break;
+                }
+                printf("offset: %lu length: %lu\n", offset, (uint64_t)buf->size);
+                if(fwrite(buf->space, 1, buf->size, out) == 0) {
+                    perror("write");
+                    error = 1;
+                    break;
+                }
+                
+                FilePart *update = calloc(1, sizeof(FilePart));
+                update->block = partnum;
+                update->etag = etag ? strdup(etag) : NULL;
+                update->hash = dav_create_hash(buf->space, buf->size);
+                updates = ucx_list_append(updates, update);
+            }
+            
+            
+            
+            i++;
+        } // else: res is not a regular file part
+        part = part->next;
+    }
+    
+    ucx_buffer_free(buf);
+    
+    if(error) {
+        *err = 1;
+        ucx_list_free_content(updates, (ucx_destructor)filepart_free);
+        return NULL;
+    }
+    
+    *err = 0;
+    *blockcount = i;
+    *truncate_file = maxsize;
+    return updates;
+}
+
+
+
 int sync_get_resource(
         CmdArgs *a,
         SyncDirectory *dir,
@@ -944,49 +1055,87 @@
         int *counter)
 { 
     LocalResource *local = ucx_map_cstr_get(db->resources, path);
-    char *local_path = util_concat_path(dir->path, path);
+    char *local_path = create_local_path(dir, path);
     
     char *etag = dav_get_string_property(res, "D:getetag");
     SYS_STAT s;
     memset(&s, 0, sizeof(SYS_STAT));
+    
+    char *blocksize_str = dav_get_string_property_ns(res, DAV_NS, "split");
+    int64_t blocksize = 0;
+    DavBool issplit = FALSE;
+    if(blocksize_str) {
+        if(!util_strtouint(blocksize_str, &blocksize)) {
+            fprintf(stderr, "Error: split property does not contain an integer.\n");
+            return 1;
+        }
+        issplit = TRUE;
+    }
+    UcxList *part_updates = NULL;
+    uint64_t blockcount = 0;
+    
+    if(res->iscollection && !issplit) {
+        // why are we here?
+        return 0;
+    }
       
     int ret = 0;
-    char *tmp_path = create_tmp_download_path(local_path);
-    
-    if(!tmp_path) {
-        fprintf(stderr, "Cannot create tmp path for %s\n", local_path);
-        free(local_path);
-        return -1;
-    }
-    FILE *out = sys_fopen(tmp_path, "wb");
+    
+    char *tmp_path = NULL;
+    FILE *out = NULL;
+    if(!issplit) {
+        tmp_path = create_tmp_download_path(local_path);
+        if(!tmp_path) {
+            fprintf(stderr, "Cannot create tmp path for %s\n", local_path);
+            free(local_path);
+            return -1;
+        }
+        out = sys_fopen(tmp_path , "wb");
+    } else {
+        out = sys_fopen(local_path, "r+b");
+        if(!out && errno == ENOENT) {
+            out = sys_fopen(local_path, "wb");
+        }
+    }
     if(!out) {
         fprintf(stderr, "Cannot open output file: %s\n", local_path);
         free(local_path);
-        free(tmp_path);
+        if(tmp_path) {
+            free(tmp_path);
+        }
         return -1;
     }
+    
+    int64_t truncate_file = -1;
     printf("get: %s\n", path);
-    if(dav_get_content(res, out, (dav_write_func)fwrite)) {
-        ret = -1;
+    if(issplit) {
+        part_updates = sync_download_changed_parts(res, local, out, blocksize, &blockcount, &truncate_file, &ret);
+    } else {
+        ret = dav_get_content(res, out, (dav_write_func)fwrite);
     }
     fclose(out);
+    if(truncate_file >= 0) {
+        truncate(tmp_path, truncate_file);
+    }
 
     if(ret == 0) {
         (*counter)++;
-
-        if(dir->trash && dir->backuppull) {
-            move_to_trash(dir, local_path);
-        }
-        if(sys_rename(tmp_path, local_path)) {
-            fprintf(
-                    stderr,
-                    "Cannot rename file %s to %s\n",
-                    tmp_path,
-                    local_path);
-            perror("");
-            free(tmp_path);
-            free(local_path);
-            return -1;
+        
+        if(tmp_path) {
+            if(dir->trash && dir->backuppull) {
+                move_to_trash(dir, local_path);
+            }
+            if(sys_rename(tmp_path, local_path)) {
+                fprintf(
+                        stderr,
+                        "Cannot rename file %s to %s\n",
+                        tmp_path,
+                        local_path);
+                perror("");
+                free(tmp_path);
+                free(local_path);
+                return -1;
+            }
         }
 
         if(sys_stat(local_path, &s)) {
@@ -1009,23 +1158,27 @@
             free(local->etag);
         }
         
+        update_parts(local, part_updates, blockcount);
+        
         // set metadata from stat
         local->etag = strdup(etag);
         sync_set_metadata_from_stat(local, &s);
         local->skipped = FALSE;
-    } else {
+    } else if(tmp_path) {
         if(sys_unlink(tmp_path)) {
             fprintf(stderr, "Cannot remove tmp file: %s\n", tmp_path);
         }
     }
     
-    free(tmp_path);
+    if(tmp_path) {
+        free(tmp_path);
+    }
     free(local_path);
     return ret;
 }
 
 int sync_remove_local_resource(SyncDirectory *dir, LocalResource *res) {
-    char *local_path = util_concat_path(dir->path, res->path);
+    char *local_path = create_local_path(dir, res->path);
     SYS_STAT s;
     if(sys_stat(local_path, &s)) {
         free(local_path);
@@ -1057,7 +1210,7 @@
 
 int sync_remove_local_directory(SyncDirectory *dir, LocalResource *res) {
     int ret = 0;
-    char *local_path = util_concat_path(dir->path, res->path);
+    char *local_path = create_local_path(dir, res->path);
     
     printf("delete: %s\n", res->path);
     if(rmdir(local_path)) {
@@ -1069,10 +1222,30 @@
     return ret;
 }
 
-void rename_conflict_file(SyncDirectory *dir, SyncDatabase *db, char *path) {
-    char *local_path = util_concat_path(dir->path, path);
+int copy_file(const char *from, const char *to) {
+    FILE *in = sys_fopen(from, "rb");
+    if(!in) {
+        return 1;
+    }
+    FILE *out = sys_fopen(to, "wb");
+    if(!out) {
+        fclose(in);
+        return 1;
+    }
+    
+    ucx_stream_copy(in, out, (read_func)fread, (write_func)fwrite);
+    fclose(in);
+    fclose(out);
+}
+
+typedef int (*renamefunc)(const char*,const char*);
+
+void rename_conflict_file(SyncDirectory *dir, SyncDatabase *db, char *path, DavBool copy) {
+    char *local_path = create_local_path(dir, path);
     char *parent = util_parent_path(local_path);
     
+    renamefunc fn = copy ? copy_file : sys_rename;
+    
     int rev = 0;
     SYS_STAT s;
     int loop = 1;
@@ -1096,7 +1269,7 @@
             if(errno == ENOENT) {
                 loop = 0;
                 printf("conflict: %s\n", local_path);
-                if(sys_rename(local_path, new_path.ptr)) {
+                if(fn(local_path, new_path.ptr)) {
                     //printf("errno: %d\n", errno);
                     fprintf(
                             stderr,
@@ -1637,7 +1810,7 @@
             }
         }
         
-        char *file_path = util_concat_path(dir->path, resource->path);
+        char *file_path = create_local_path(dir, resource->path);
         SYS_STAT s;
         if(sys_stat(file_path, &s)) {
             if(errno == ENOENT) {
@@ -1741,7 +1914,7 @@
         // download the resource
         if(!sync_shutdown) {
             if(resource->isdirectory) {
-                char *local_path = util_concat_path(dir->path, res->path);
+                char *local_path = create_local_path(dir, res->path);
                 if(sys_mkdir(local_path) && errno != EEXIST) {
                     fprintf(stderr,
                             "Cannot create directory %s: %s",
@@ -1803,7 +1976,7 @@
         
         char *p = stack->data;
         stack = ucx_list_remove(stack, stack);
-        char *local_path = util_concat_path(dir->path, p);
+        char *local_path = create_local_path(dir, p);
         SYS_DIR local_dir = sys_opendir(local_path);
         
         if(!local_dir) {
@@ -1885,7 +2058,7 @@
 }
 
 LocalResource* local_resource_new(SyncDirectory *dir, SyncDatabase *db, char *path, int *isdir) {
-    char *file_path = util_concat_path(dir->path, path);
+    char *file_path = create_local_path(dir, path);
     SYS_STAT s;
     if(sys_stat(file_path, &s)) {
         fprintf(stderr, "Cannot stat file %s\n", file_path);
@@ -2011,7 +2184,7 @@
         }
         
         if((dir->metadata & FINFO_XATTR) == FINFO_XATTR) {
-            char *path = util_concat_path(dir->path, db_res->path);
+            char *path = create_local_path(dir, db_res->path);
             XAttributes *xattr = file_get_attributes(path);
             // test if xattr are added, removed or changed
             if((db_res->xattr_hash && !xattr) ||
@@ -2448,7 +2621,7 @@
     UcxBuffer *buf = NULL;
     if(dir->tagconfig->store == TAG_STORE_XATTR) {
         ssize_t tag_length = 0;
-        char *local_path = util_concat_path(dir->path, res->path);
+        char *local_path = create_local_path(dir, res->path);
         char* tag_data = xattr_get(
                 local_path,
                 dir->tagconfig->xattr_name,
@@ -2886,7 +3059,7 @@
     return updated_parts;
 }
 
-static void update_parts(LocalResource *local, UcxList *updates, uint64_t numparts) {
+void update_parts(LocalResource *local, UcxList *updates, uint64_t numparts) {
     size_t old_num = local->numparts;
     if(old_num > numparts) {
         // free old parts
@@ -2938,7 +3111,7 @@
         LocalResource *local,
         int *counter)
 {
-    char *local_path = util_concat_path(dir->path, res->path);
+    char *local_path = create_local_path(dir, res->path);
     
     SYS_STAT s;
     if(sys_stat(local_path, &s)) {
@@ -3265,7 +3438,7 @@
     UcxMapIterator i = ucx_map_iterator(db->conflict);
     LocalResource *res;
     UCX_MAP_FOREACH(key, res, i) {
-        char *path = util_concat_path(dir->path, res->path);
+        char *path = create_local_path(dir, res->path);
         SYS_STAT s;
         if(sys_stat(path, &s)) {
             if(errno == ENOENT) {
@@ -3386,7 +3559,7 @@
     LocalResource *res;
     UCX_MAP_FOREACH(key, res, i) {
         printf("delete: %s\n", res->path);
-        char *path = util_concat_path(dir->path, res->path);
+        char *path = create_local_path(dir, res->path);
         if(sys_unlink(path)) {
             if(errno != ENOENT) {
                 perror("unlink");
--- a/dav/sync.h	Thu Mar 28 14:18:54 2019 +0100
+++ b/dav/sync.h	Thu Mar 28 18:09:31 2019 +0100
@@ -73,7 +73,7 @@
     REMOTE_CHANGE_DELETED,
     REMOTE_CHANGE_CONFLICT_LOCAL_MODIFIED,
     REMOTE_CHANGE_METADATA,
-    REMOTE_CHANGE_MKDIR
+    REMOTE_CHANGE_MKDIR,
 };
 typedef enum RemoteChangeType RemoteChangeType;
 
@@ -81,7 +81,7 @@
     DavResource      *resource;
     RemoteChangeType type;
 } RemoteChange;
-    
+
 void print_usage(char *cmd);
 
 pthread_t start_sighandler(pthread_mutex_t *mutex) ;
@@ -109,7 +109,7 @@
         int *counter);
 int sync_remove_local_resource(SyncDirectory *dir, LocalResource *res);
 int sync_remove_local_directory(SyncDirectory *dir, LocalResource *res);
-void rename_conflict_file(SyncDirectory *dir, SyncDatabase *db, char *path);
+void rename_conflict_file(SyncDirectory *dir, SyncDatabase *db, char *path, DavBool copy);
 char* create_tmp_download_path(char *path);
 void move_to_trash(SyncDirectory *dir, char *path);
 
@@ -165,6 +165,8 @@
         DavResource *res,
         LocalResource *local);
 
+void update_parts(LocalResource *local, UcxList *updates, uint64_t numparts);
+
 void remove_deleted_conflicts(SyncDirectory *dir, SyncDatabase *db);
 
 int cmd_resolve_conflicts(CmdArgs *args);
--- a/libidav/utils.c	Thu Mar 28 14:18:54 2019 +0100
+++ b/libidav/utils.c	Thu Mar 28 18:09:31 2019 +0100
@@ -863,3 +863,14 @@
     ucx_buffer_free(buf);
     return str;
 }
+
+void util_remove_trailing_pathseparator(char *path) {
+    size_t len = strlen(path);
+    if(len < 2) {
+        return;
+    }
+    
+    if(path[len-1] == '/') {
+        path[len-1] = '\0';
+    }
+}
--- a/libidav/utils.h	Thu Mar 28 14:18:54 2019 +0100
+++ b/libidav/utils.h	Thu Mar 28 18:09:31 2019 +0100
@@ -109,6 +109,8 @@
 
 char* util_hexstr(const unsigned char *data, size_t len);
 
+void util_remove_trailing_pathseparator(char *path);
+
 #ifdef	__cplusplus
 }
 #endif

mercurial