# HG changeset patch
# User Olaf Wintermann <olaf.wintermann@gmail.com>
# Date 1548432119 -3600
# Node ID f84e64afee61502d6e44f43ab27195a1426bb40a
# Parent  a23fedac340c555e970fb3229cfe5f4768e1d389
dav-sync pull refactoring: create list of changes before sync

diff -r a23fedac340c -r f84e64afee61 dav/sopt.h
--- a/dav/sopt.h	Tue Jan 08 16:37:13 2019 +0100
+++ b/dav/sopt.h	Fri Jan 25 17:01:59 2019 +0100
@@ -35,6 +35,8 @@
 extern "C" {
 #endif
 
+typedef void* ArgBool;
+    
 typedef struct {
     UcxMap *options;
     char   **argv;
diff -r a23fedac340c -r f84e64afee61 dav/sync.c
--- a/dav/sync.c	Tue Jan 08 16:37:13 2019 +0100
+++ b/dav/sync.c	Fri Jan 25 17:01:59 2019 +0100
@@ -525,12 +525,16 @@
     int sync_delete = 0;
     int sync_error = 0;
     
-    UcxList *ls_get = NULL;
-    UcxList *ls_remove = NULL;
-    UcxList *ls_conflict = NULL;
-    UcxList *ls_broken = NULL;
+    UcxList *res_modified = NULL;
+    UcxList *res_new = NULL;
+    UcxList *res_conflict = NULL;
+    UcxList *res_mkdir = NULL;
+    UcxList *res_metadata = NULL;
+    UcxList *res_broken = NULL;
+    UcxList *lres_removed = NULL; // list of LocalResource*
     
     UcxMap *svrres = ucx_map_new(db->resources->count);
+    UcxMap *dbres = ucx_map_clone(db->resources, NULL, NULL);
     
     UcxList *statls = NULL;
     
@@ -571,28 +575,41 @@
             if(status && !strcmp(status, "broken")) {
                 res = res->next;
                 localres_keep(db, res->path);
-                ls_broken = ucx_list_append(ls_broken, res);
+                res_broken = ucx_list_append(res_broken, res);
                 continue;
             }
             
-            // download the resource
-            if(!sync_shutdown && sync_get_resource(a, dir, res, db, FALSE, &sync_success)) {
-                fprintf(stderr, "sync_get_resource failed for resource: %s\n", res->path);
-                sync_error++;
+            // check if a resource has changed on the server
+            int change = resource_get_remote_change(a, res, dir, db);
+            switch(change) {
+                case REMOTE_NO_CHANGE: break;
+                case REMOTE_CHANGE_MODIFIED: {
+                    res_modified = ucx_list_append(res_modified, res);
+                    break;
+                }
+                case REMOTE_CHANGE_NEW: {
+                    res_new = ucx_list_append(res_new, res);
+                    break;
+                }
+                case REMOTE_CHANGE_DELETED: break; // never happens
+                case REMOTE_CHANGE_CONFLICT_LOCAL_MODIFIED: {
+                    res_conflict = ucx_list_append(res_conflict, res);
+                    break;
+                }
+                case REMOTE_CHANGE_METADATA: {
+                    res_metadata = ucx_list_append(res_metadata, res);
+                    break;
+                }
+                case REMOTE_CHANGE_MKDIR: {
+                    res_mkdir = ucx_list_append(res_mkdir, res);
+                    break;
+                }
             }
             
-            // add every resource from the server to svrres
-            // then db-resources contains only resources which are not on the
-            // server
-            LocalResource *local = ucx_map_cstr_remove(db->resources, res->path);
-            if(local) {
-                ucx_map_cstr_put(svrres, res->path, local);
-                
-                if(local->last_modified == 0) {
-                    // stat this file later (directory)
-                    statls = ucx_list_prepend(statls, local);
-                }
-            } // else: sync_shutdown is TRUE
+            // remove every server resource from dbres
+            // all remaining elements are the resources that are removed
+            // on the server
+            ucx_map_cstr_remove(dbres, res->path);
             
             if(res->children) {
                 stack = ucx_list_prepend(stack, res->children);
@@ -601,46 +618,109 @@
         }
     }
     
-    // stat all files with unknown lastmodified date
-    UCX_FOREACH(elm, statls) {
-        LocalResource *l = elm->data;
-        char *local_path = util_concat_path(dir->path, l->path);
-        SYS_STAT s;
-        if(!sys_stat(local_path, &s)) {
-            l->last_modified = s.st_mtime;
-        }
-        free(local_path);
-    }
-    ucx_list_free(statls);
-    
-    // delete every remotely removed resource
-    UcxMapIterator i = ucx_map_iterator(db->resources);
+    // find deleted resources
+    // svrres currently contains all resources from the server
+    // and will replace the current db->resources map later
+    UcxMapIterator i = ucx_map_iterator(dbres);
     LocalResource *local;
-    UcxList *rmdirs = NULL;
     UCX_MAP_FOREACH(key, local, i) {
         if (res_matches_filter(dir, local->path)) {
             continue;
         }
+        if(!local->keep) {
+            lres_removed = ucx_list_prepend(lres_removed, local);
+        }
+    }
+    
+    // the first thing we need are all directories to put the files in
+    UCX_FOREACH(elm, res_mkdir) {
+        DavResource *res = elm->data;
+        char *local_path = util_concat_path(dir->path, res->path);
+        if(sys_mkdir(local_path) && errno != EEXIST) {
+            fprintf(stderr,
+                    "Cannot create directory %s: %s",
+                    local_path, strerror(errno));
+        }
+        free(local_path);
+    }
+    
+    // we need a map for all conflicts for fast lookups
+    UcxMap *conflicts = ucx_map_new(ucx_list_size(res_conflict)+16);
+    UCX_FOREACH(elm, res_conflict) {
+        DavResource *res = elm->data;
+        ucx_map_cstr_put(conflicts, res->path, res);
+    }
+    
+    // download all new, modified and conflict files
+    UcxList *download = ucx_list_concat(res_modified, res_conflict);
+    download = ucx_list_concat(res_new, download);
+    UCX_FOREACH(elm, download) {
+        DavResource *res = elm->data;
+        if(sync_shutdown) {
+            break;
+        }
         
-        if(sync_shutdown || local->keep) {
-            ucx_map_cstr_put(svrres, local->path, local_resource_copy(local));
-        } else {
-            // sync_remove_resource does all necessary tests
-            int ret = sync_remove_local_resource(dir, local);
-            if(ret == -1) {
-                rmdirs = ucx_list_append(rmdirs, local);
-            } else if(ret == 0) {
-                sync_delete++;
-            }
+        if(ucx_map_cstr_get(conflicts, res->path)) {
+            rename_conflict_file(dir, db, res->path);
+        }
+        
+        // download the resource
+        if(sync_get_resource(a, dir, res, db, &sync_success)) {
+            fprintf(stderr, "resource download failed: %s\n", res->path);
+            sync_error++;
         }
     }
+    
+    UCX_FOREACH(elm, res_metadata) {
+        DavResource *res = elm->data;
+        if(sync_shutdown) {
+            break;
+        }
+        
+        LocalResource *local = ucx_map_cstr_get(db->resources, res->path);
+        if(local) {
+            char *local_path = util_concat_path(dir->path, res->path);
+            if(sync_store_tags(dir, local_path, local, res)) {
+                fprintf(stderr, "Tag update failed: %s\n", res->path);
+            }
+            free(local_path);
+        } else {
+            // this should never happen but who knows
+            fprintf(stderr,
+                    "Cannot update metadata of file %s: not in database\n",
+                    res->path);
+        }
+    }
+    
+    UcxList *rmdirs = NULL;
+    UCX_FOREACH(elm, lres_removed) {
+        LocalResource *res = elm->data;
+        if(sync_shutdown) {
+            break;
+        }
+        
+        int ret = sync_remove_local_resource(dir, res);
+        if(ret == -1) {
+            rmdirs = ucx_list_append(rmdirs, res);
+        } else if(ret == 0) {
+            LocalResource *local = ucx_map_cstr_remove(db->resources, res->path);
+            if(local) {
+                local_resource_free(local);
+            }
+            sync_delete++;
+        }     
+    }
+    
     UCX_FOREACH(elm, rmdirs) {
         LocalResource *local_dir = elm->data;
-        sync_remove_local_directory(dir, local_dir);
+        if(!sync_remove_local_directory(dir, local_dir)) {
+            LocalResource *local = ucx_map_cstr_remove(db->resources, local_dir->path);
+            if(local) {
+                local_resource_free(local);
+            }
+            sync_delete++;
+        }
     }
-    ucx_map_free_content(db->resources, (ucx_destructor)local_resource_free);
-    ucx_map_free(db->resources);
-    db->resources = svrres;
     
     // unlock repository
     if(locked) {
@@ -679,26 +759,68 @@
     return ret;
 }
 
-int sync_remote_is_changed(
+
+RemoteChangeType resource_get_remote_change(
         CmdArgs *a,
-        SyncDirectory *dir,
         DavResource *res,
-        SyncDatabase *db,
-        DavBool force,
-        DavBool *conflict,
-        DavBool *metadataupdate)
+        SyncDirectory *dir,
+        SyncDatabase *db)
 {
-    int cdt = cmd_getoption(a, "conflict") ? 0 : 1; // conflict detection
-    *conflict = 0;
-    *metadataupdate = 0;
+    char *etag = dav_get_string_property(res, "D:getetag");
+    if(!etag) {
+        fprintf(stderr, "Error: resource %s has no etag\n", res->path);
+        return REMOTE_NO_CHANGE;
+    }
+    
+    RemoteChangeType type = cmd_getoption(a, "conflict") ?
+            REMOTE_CHANGE_MODIFIED : REMOTE_CHANGE_CONFLICT_LOCAL_MODIFIED;
     
     LocalResource *local = ucx_map_cstr_get(db->resources, res->path);
     char *local_path = util_concat_path(dir->path, res->path);
     
-    char *etag = dav_get_string_property(res, "D:getetag");
     SYS_STAT s;
-    int ret = 0;
+    DavBool exists = 1;
+    if(sys_stat(local_path, &s)) {
+        if(errno != ENOENT) {
+            fprintf(stderr, "Cannot stat file: %s\n", local_path);
+            free(local_path);
+            return REMOTE_NO_CHANGE;
+        }
+        exists = 0;
+    }
     
+    RemoteChangeType ret = REMOTE_NO_CHANGE;
+    if(res->iscollection) {
+        if(!exists) {
+            ret = REMOTE_CHANGE_MKDIR;
+        }
+    } else if(local) {
+        DavBool nochange = FALSE;
+        if(local->etag) {
+            sstr_t e = sstr(etag);
+            if(sstrprefix(e, S("W/"))) {
+                e = sstrsubs(e, 2);
+            }
+            if(!strcmp(e.ptr, local->etag)) {
+                // resource is already up-to-date on the client
+                
+                // TODO: detect metadata update
+                //sync_store_tags(dir, local_path, local, res);
+                nochange = TRUE;
+            }
+        }
+        
+        if(!nochange) {
+            if(!(exists && s.st_mtime != local->last_modified)) {
+                type = REMOTE_CHANGE_MODIFIED;
+            }
+            ret = type;
+        }
+    } else if(exists) {
+        ret = type;
+    } else {
+        ret = REMOTE_CHANGE_NEW;
+    }
     
     free(local_path);
     return ret;
@@ -709,149 +831,79 @@
         SyncDirectory *dir,
         DavResource *res,
         SyncDatabase *db,
-        DavBool force,
         int *counter)
-{
-    int cdt = cmd_getoption(a, "conflict") ? 0 : 1; // conflict detection
-    
+{ 
     LocalResource *local = ucx_map_cstr_get(db->resources, res->path);
     char *local_path = util_concat_path(dir->path, res->path);
     
     char *etag = dav_get_string_property(res, "D:getetag");
     SYS_STAT s;
     memset(&s, 0, sizeof(SYS_STAT));
-    if(!force) {
-        if(local && !res->iscollection) {
-            int exists = 1;
-            if(sys_stat(local_path, &s)) {
-                // Ignore the fact, that the file is locally removed. If the
-                // server has an updated version, we read the file or the
-                // next push will delete it on the server.
-                if(errno != ENOENT) {
-                    fprintf(stderr, "Cannot stat file: %s\n", local_path);
-                    free(local_path);
-                    return -1;
-                } else {
-                    exists = 0;
-                }
-            }
-
-            if(local->etag) {
-                sstr_t e = sstr(etag);
-                if(sstrprefix(e, S("W/"))) {
-                    e = sstrsubs(e, 2);
-                }
-                if(!strcmp(e.ptr, local->etag)) {
-                    // resource is already up-to-date on the client
-                    sync_store_tags(dir, local_path, local, res);
-                    free(local_path);
-                    return 0;
-                }
-            }
-
-            if(cdt && exists && s.st_mtime != local->last_modified) {
-                // file modified on the server and on the client
-                rename_conflict_file(dir, db, local->path);
-            }
-        } else {
-            if(sys_stat(local_path, &s)) {
-                if(errno != ENOENT) {
-                    fprintf(stderr, "Cannot stat file: %s\n", local_path);
-                }
-            } else if(S_ISDIR(s.st_mode)) {
-                //fprintf(stderr, "Error: file %s is a directory\n", local_path);
-            } else if(cdt) {
-                // rename file on conflict
-                rename_conflict_file(dir, db, res->path);
-            }
-        }
-    }
       
     int ret = 0;
     char *tmp_path = create_tmp_download_path(local_path);
-    if(res->iscollection) {
-        if(sys_mkdir(local_path) && errno != EEXIST) {
-            ret = -1;
+    
+    if(!tmp_path) {
+        fprintf(stderr, "Cannot create tmp path for %s\n", local_path);
+        free(local_path);
+        return -1;
+    }
+    FILE *out = sys_fopen(tmp_path, "wb");
+    if(!out) {
+        fprintf(stderr, "Cannot open output file: %s\n", local_path);
+        free(local_path);
+        free(tmp_path);
+        return -1;
+    }
+    printf("get: %s\n", res->path);
+    if(dav_get_content(res, out, (dav_write_func)fwrite)) {
+        ret = -1;
+    }
+    fclose(out);
+
+    if(ret == 0) {
+        (*counter)++;
+
+        if(dir->trash && dir->backuppull) {
+            move_to_trash(dir, local_path);
         }
-        
-        if(ret == 0) {
-            if(!local) {
-                // new local resource
-                local = calloc(1, sizeof(LocalResource));
-                local->path = util_concat_path(res->path, "/");
-                local->last_modified = 0;
-                if(local->etag) {
-                    free(local->etag);
-                }
-                local->etag = strdup(etag);
-                ucx_map_cstr_put(db->resources, local->path, local);
-            }
-            
-            sync_store_tags(dir, local_path, local, res);
-        }
-    } else {
-        if(!tmp_path) {
-            fprintf(stderr, "Cannot create tmp path for %s\n", local_path);
+        if(sys_rename(tmp_path, local_path)) {
+            fprintf(
+                    stderr,
+                    "Cannot rename file %s to %s\n",
+                    tmp_path,
+                    local_path);
+            perror("");
+            free(tmp_path);
             free(local_path);
             return -1;
         }
-        FILE *out = sys_fopen(tmp_path, "wb");
-        if(!out) {
-            fprintf(stderr, "Cannot open output file: %s\n", local_path);
-            free(local_path);
-            free(tmp_path);
-            return -1;
+
+        if(sys_stat(local_path, &s)) {
+            fprintf(stderr,
+                    "Cannot stat file %s: %s\n", local_path, strerror(errno));
         }
-        printf("get: %s\n", res->path);
-        if(dav_get_content(res, out, (dav_write_func)fwrite)) {
-            ret = -1;
+
+        if(!local) {
+            // new local resource
+            local = calloc(1, sizeof(LocalResource));
+            local->path = strdup(res->path);
+            ucx_map_cstr_put(db->resources, local->path, local);
         }
-        fclose(out);
+
+        if(local->etag) {
+            free(local->etag);
+        }
+        // set metadata from stat
+        local->etag = strdup(etag);
+        local->last_modified = s.st_mtime;
+        local->size = s.st_size;
+        local->skipped = FALSE;
         
         sync_store_tags(dir, tmp_path, local, res);
-        
-        if(ret == 0) {
-            (*counter)++;
-            
-            if(dir->trash && dir->backuppull) {
-                move_to_trash(dir, local_path);
-            }
-            if(sys_rename(tmp_path, local_path)) {
-                fprintf(
-                        stderr,
-                        "Cannot rename file %s to %s\n",
-                        tmp_path,
-                        local_path);
-                perror("");
-                free(tmp_path);
-                free(local_path);
-                return -1;
-            }
-            
-            if(sys_stat(local_path, &s)) {
-                fprintf(stderr, "Cannot stat file: %s\n", local_path);
-                perror("");
-            }
-            
-            if(!local) {
-                // new local resource
-                local = calloc(1, sizeof(LocalResource));
-                local->path = strdup(res->path);
-                ucx_map_cstr_put(db->resources, local->path, local);
-            }
-            
-            if(local->etag) {
-                free(local->etag);
-            }
-            // set metadata from stat
-            local->etag = strdup(etag);
-            local->last_modified = s.st_mtime;
-            local->size = s.st_size;
-            local->skipped = FALSE;
-        } else {
-            if(sys_unlink(tmp_path)) {
-                fprintf(stderr, "Cannot remove tmp file: %s\n", tmp_path);
-            }
+    } else {
+        if(sys_unlink(tmp_path)) {
+            fprintf(stderr, "Cannot remove tmp file: %s\n", tmp_path);
         }
     }
     
@@ -891,16 +943,18 @@
     return ret;
 }
 
-void sync_remove_local_directory(SyncDirectory *dir, LocalResource *res) {
+int sync_remove_local_directory(SyncDirectory *dir, LocalResource *res) {
+    int ret = 0;
     char *local_path = util_concat_path(dir->path, res->path);
     
     printf("delete: %s\n", res->path);
     if(rmdir(local_path)) {
-        fprintf(stderr, "rmdir: %s : ", local_path);
-        perror(NULL);
+        fprintf(stderr, "rmdir: %s : %s", local_path, strerror(errno));
+        ret = 1;
     }
     
     free(local_path);
+    return ret;
 }
 
 void rename_conflict_file(SyncDirectory *dir, SyncDatabase *db, char *path) {
@@ -1474,7 +1528,7 @@
         }
         
         // download the resource
-        if(!sync_shutdown && sync_get_resource(a, dir, res, db, TRUE, &sync_success)) {
+        if(!sync_shutdown && sync_get_resource(a, dir, res, db, &sync_success)) {
             fprintf(stderr, "sync_get_resource failed for resource: %s\n", res->path);
             sync_error++;
         }
diff -r a23fedac340c -r f84e64afee61 dav/sync.h
--- a/dav/sync.h	Tue Jan 08 16:37:13 2019 +0100
+++ b/dav/sync.h	Fri Jan 25 17:01:59 2019 +0100
@@ -59,6 +59,22 @@
     SyncDirectory *dir;
     char *path;
 } SyncFile;
+
+enum RemoteChangeType {
+    REMOTE_NO_CHANGE = 0,
+    REMOTE_CHANGE_MODIFIED,
+    REMOTE_CHANGE_NEW,
+    REMOTE_CHANGE_DELETED,
+    REMOTE_CHANGE_CONFLICT_LOCAL_MODIFIED,
+    REMOTE_CHANGE_METADATA,
+    REMOTE_CHANGE_MKDIR
+};
+typedef enum RemoteChangeType RemoteChangeType;
+
+typedef struct RemoteChange {
+    DavResource      *resource;
+    RemoteChangeType type;
+} RemoteChange;
     
 void print_usage(char *cmd);
 
@@ -71,15 +87,20 @@
 int cmd_push(CmdArgs *args, DavBool archive);
 int cmd_restore(CmdArgs *args);
 
+RemoteChangeType resource_get_remote_change(
+        CmdArgs *a,
+        DavResource *res,
+        SyncDirectory *dir,
+        SyncDatabase *db);
+
 int sync_get_resource(
         CmdArgs *a,
         SyncDirectory *dir,
         DavResource *res,
         SyncDatabase *db,
-        DavBool force,
         int *counter);
 int sync_remove_local_resource(SyncDirectory *dir, LocalResource *res);
-void sync_remove_local_directory(SyncDirectory *dir, LocalResource *res);
+int sync_remove_local_directory(SyncDirectory *dir, LocalResource *res);
 void rename_conflict_file(SyncDirectory *dir, SyncDatabase *db, char *path);
 char* create_tmp_download_path(char *path);
 void move_to_trash(SyncDirectory *dir, char *path);