# HG changeset patch
# User Olaf Wintermann <olaf.wintermann@gmail.com>
# Date 1544376734 -3600
# Node ID 868da3f76267cd2b5fa8ba09f0a3a5391d156ffa
# Parent  0fe1514667e62462ec3d89e7cbac6c33930cfd3b
dav-sync push refactoring: create list of changes before executing sync

diff -r 0fe1514667e6 -r 868da3f76267 dav/sync.c
--- a/dav/sync.c	Fri Dec 07 11:48:55 2018 +0100
+++ b/dav/sync.c	Sun Dec 09 18:32:14 2018 +0100
@@ -1087,180 +1087,206 @@
     int sync_skipped = 0;
     int sync_error = 0;
     
+    UcxList *ls_put = NULL;
+    UcxList *ls_conflict = NULL;
+    UcxList *ls_update = NULL;
+    UcxList *ls_delete = NULL;
+    
     // upload all changed files
     //UcxList *resources = cmd_getoption(a, "read") ?
     //        read_changes(dir, db) : local_scan(dir, db);
     UcxList *resources = local_scan(dir, db);
-       
-    UcxMap *lclres = ucx_map_new(db->resources->count);
-    int ret = 0;
+    UcxMap *resources_map = ucx_map_new(ucx_list_size(resources)+16);
+    
     UCX_FOREACH(elm, resources) {
         LocalResource *local_res = elm->data;
-        int error = 0;
         
-        DavBool res_filtered = FALSE;
+        // ignore all files, that are excluded by a static filter (sync.xml) 
+        
+        // static include/exclude filter
         if(res_matches_filter(dir, local_res->path+1)) {
-            res_filtered = TRUE;
-        } else {
-            UCX_FOREACH(elm, dir->tagfilter) {
-                SyncTagFilter *tf = elm->data;
-                if(!localres_matches_tags(dir, local_res, tf)) {
-                    res_filtered = TRUE;
-                    break;
-                }
+            continue;
+        }
+        // static tag filter
+        UCX_FOREACH(elm, dir->tagfilter) {
+            SyncTagFilter *tf = elm->data;
+            if(!localres_matches_tags(dir, local_res, tf)) {
+                continue;
             }
         }
-        if (res_filtered) {
-            local_res->keep = TRUE;
-        } else if (!localres_matches_tags(dir, local_res, tagfilter)) {
+        
+        // we need a fast file lookup map later
+        ucx_map_cstr_put(resources_map, local_res->path, local_res);
+        
+        // dynamic tag filter
+        if(!localres_matches_tags(dir, local_res, tagfilter)) {
             if(!remove_file) {
-                local_res->keep = TRUE;
+                LocalResource *dbres = ucx_map_cstr_get(
+                        db->resources,
+                        local_res->path);
+                if(dbres) {
+                    // this makes sure the file will not be deleted later
+                    dbres->keep = TRUE;
+                }
             }
-        } else {
-            if(sync_shutdown) {
-                LocalResource *lr = ucx_map_cstr_remove(db->resources, local_res->path);
-                if(lr) {
-                    local_res->size = lr->size;
-                    local_res->last_modified = lr->last_modified;
-                    local_res->etag = lr->etag ? strdup(lr->etag) : NULL;
-                    local_resource_free(lr);
-                    ucx_map_cstr_put(lclres, local_res->path, local_res);
-                }
-                elm->data = NULL;
-                continue;
-            }
-            
-            
-            if(res_isconflict(db, local_res)) {
-                printf("skip: %s\n", local_res->path);
-                sync_skipped++;
-                continue;
-            }
-                       
-            // upload every changed file
-            int is_changed = local_resource_is_changed(
+            continue;
+        }
+        
+        if(res_isconflict(db, local_res)) {
+            ls_conflict = ucx_list_append(ls_conflict, local_res);
+            continue;
+        }
+        
+        int is_changed = local_resource_is_changed(
                     dir,
                     db,
                     local_res,
                     svrres,
                     restore_removed,
                     restore_modified);
-            if (is_changed || local_res->tags_updated) {
-                DavResource *res = dav_resource_new(sn, local_res->path);
-                if(!res) {
-                    print_resource_error(sn, local_res->path);
-                    ret = -1;
-                    sync_error++;
-                }
-                
-                if(local_res->isdirectory) {
-                    dav_exists(res);
-                    if(sn->error == DAV_NOT_FOUND) {
-                        int abort = 0;
-                        // make sure to store tags for newly created cols
-                        local_res->tags_updated = 1;
-                        // create collection
-                        // TODO: show 405
-                        printf("mkcol: %s\n", local_res->path);
-                        if(sync_mkdir(dir, res, local_res) && sn->error != DAV_METHOD_NOT_ALLOWED) {
-                            print_resource_error(sn, res->path);
-                            ret = -1;
-                            sync_error++;
-                            error = 1;
-                            abort = 1;
-                        }
-                        
-                        if(dir->tagconfig && local_res->tags_updated && !abort) {
-                            sync_update_tags(dir, sn, res, local_res);
-                        }
-                    } else if(sn->error != DAV_OK) {
-                        // dav_exists() failed
-                        print_resource_error(sn, local_res->path);
-                        ret = -1;
-                        sync_error++;
-                        error = 1;
-                    }
-                } else if(!is_changed) {
-                    if(dir->tagconfig && local_res->tags_updated) {
-                        sync_update_tags(dir, sn, res, local_res);
-                    }
-                } else {
-                    if(cdt && remote_resource_is_changed(sn, dir, db, local_res)) {
-                        printf("conflict: %s\n", local_res->path);
-                        local_res->last_modified = 0;
-                        local_res->skipped = TRUE;
-                        sync_skipped++;
-                    } else {
-                        printf("put: %s\n", local_res->path);
-                        if(sync_put_resource(dir, res, local_res, &sync_success)) {
-                            sync_error++;
-                            print_resource_error(sn, res->path);
-                            ret = -1;
-                            error = 1;
-                        }
-                    }
-                }
-                dav_resource_free(res);
+        if(is_changed) {
+            ls_put = ucx_list_append(ls_put, local_res);
+        } else if(local_res->tags_updated) {
+            ls_update = ucx_list_append(ls_update, local_res);
+        }
+    }
+    
+    // find all deleted files and cleanup the database
+    UcxMapIterator i = ucx_map_iterator(db->resources);
+    LocalResource *local;
+    UCX_MAP_FOREACH(key, local, i) {
+        // all filtered files should be removed from the database
+        if(res_matches_filter(dir, local->path+1)) {
+            ucx_map_cstr_remove(db->resources, local->path);
+            continue;
+        }
+        UCX_FOREACH(elm, dir->tagfilter) {
+            SyncTagFilter *tf = elm->data;
+            if(!localres_matches_tags(dir, local, tf)) {
+                ucx_map_cstr_remove(db->resources, local->path);
+                continue;
             }
         }
         
-        // remove every locally available resource from db->resource
-        // the remaining elements are all deleted files
-        elm->data = NULL;
-        if(!error) {
-            ucx_map_cstr_put(lclres, local_res->path, local_res);
-        }
-
-        LocalResource *lr = ucx_map_cstr_remove(db->resources, local_res->path);
-        if(lr) {
-            local_resource_free(lr);
+        if(!ucx_map_get(resources_map, key)) {
+            // The current LocalResource is in the database but doesn't exist
+            // in the filesystem anymore. This means the file was deleted
+            // and should be deleted on server
+            if(!archive) {
+                ls_delete = ucx_list_append(ls_delete, local);
+            } else {
+                ucx_map_cstr_remove(db->resources, local->path);
+            }
         }
     }
     
-    // delete all removed files
-    if(ret == 0 && !archive) {
-        UcxList *cols = NULL;
+    ls_delete = ucx_list_sort(ls_delete, (cmp_func)resource_pathlen_cmp, NULL);
+    
+    //
+    // BEGIN PUSH
+    //
+    
+    // upload changed files 
+    int ret = 0;
+    int error = 0;
+    for(UcxList *elm=ls_put;elm && !sync_shutdown;elm=elm->next) {
+        LocalResource *local_res = elm->data;
+        
+        DavResource *res = dav_resource_new(sn, local_res->path);
+        if(!res) {
+            print_resource_error(sn, local_res->path);
+            ret = -1;
+            sync_error++;
+        }
         
-        UcxMapIterator i = ucx_map_iterator(db->resources);
-        LocalResource *local;
-        UCX_MAP_FOREACH(key, local, i) {
-            if (!local->keep && !res_matches_filter(dir, local->path+1)) {
-                if(sync_shutdown) {
-                    ucx_map_cstr_put(lclres, local->path, local_resource_copy(local));
-                } else if(sync_delete_remote_resource(dir, sn, local, &sync_delete, &cols)) {
-                    ucx_map_cstr_put(lclres, local->path, local_resource_copy(local));
-                    if(sn->error != DAV_NOT_FOUND) {
-                        print_resource_error(sn, local->path);
-                        sync_error++;
-                        break;
-                    }
+        if(local_res->isdirectory) {
+            dav_exists(res);
+            if(sn->error == DAV_NOT_FOUND) {
+                int abort = 0;
+                // make sure to store tags for newly created cols
+                local_res->tags_updated = 1;
+                // create collection
+                // TODO: show 405
+                printf("mkcol: %s\n", local_res->path);
+                if(sync_mkdir(dir, res, local_res) && sn->error != DAV_METHOD_NOT_ALLOWED) {
+                    print_resource_error(sn, res->path);
+                    ret = -1;
+                    sync_error++;
+                    error = 1;
+                    abort = 1;
+                }
+
+                if(dir->tagconfig && local_res->tags_updated && !abort) {
+                    sync_update_tags(dir, sn, res, local_res);
+                }
+            } else if(sn->error != DAV_OK) {
+                // dav_exists() failed
+                print_resource_error(sn, local_res->path);
+                ret = -1;
+                sync_error++;
+                error = 1;
+            }
+        } else {
+            if(cdt && remote_resource_is_changed(sn, dir, db, local_res)) {
+                printf("conflict: %s\n", local_res->path);
+                local_res->last_modified = 0;
+                local_res->skipped = TRUE;
+                sync_skipped++;
+            } else {
+                printf("put: %s\n", local_res->path);
+                if(sync_put_resource(dir, res, local_res, &sync_success)) {
+                    sync_error++;
+                    print_resource_error(sn, res->path);
+                    ret = -1;
+                    error = 1;
                 }
             }
         }
+        dav_resource_free(res);
         
-        cols = ucx_list_sort(cols, (cmp_func)resource_pathlen_cmp, NULL);
+        LocalResource *dbres = ucx_map_cstr_remove(db->resources, local_res->path);
+        ucx_map_cstr_put(db->resources, local_res->path, local_res);
+        //if(dbres) local_resource_free(dbres);
+    }
+    
+    // metadata updates
+    for(UcxList *elm=ls_update;elm && !sync_shutdown;elm=elm->next) {
+        LocalResource *local_res = elm->data;
+        
+        DavResource *res = dav_resource_new(sn, local_res->path);
         
-        UCX_FOREACH(elm, cols) {
-            local = elm->data;
-            if (!local->keep && !res_matches_filter(dir, local->path+1)) {
-                if(sync_shutdown) {
-                    ucx_map_cstr_put(lclres, local->path, local_resource_copy(local));
-                } else if(sync_delete_remote_resource(dir, sn, local, &sync_delete, NULL)) {
-                    ucx_map_cstr_put(lclres, local->path, local_resource_copy(local));
-                    if(sn->error != DAV_NOT_FOUND) {
-                        print_resource_error(sn, local->path);
-                        sync_error++;
-                        break;
-                    }
+        if(dir->tagconfig && local_res->tags_updated) {
+            sync_update_tags(dir, sn, res, local_res);
+        }
+    }  
+
+    // delete all removed files
+    UcxList *cols = NULL;
+    UcxList **col_list = &cols;
+    UcxList *deletelist = ls_delete;
+    for(int i=0;i<2;i++) {
+        // the first iteration deletes everything from ls_delete except
+        // all collections, which are stored in cols
+        // in the second run all collections will be deleted
+        for(UcxList *elm=deletelist;elm && !sync_shutdown;elm=elm->next) {
+            LocalResource *local = elm->data;
+            if(local->keep) {
+                continue;
+            }
+            if(sync_delete_remote_resource(dir, sn, local, &sync_delete, col_list)) {
+                if(sn->error != DAV_NOT_FOUND) {
+                    print_resource_error(sn, local->path);
+                    sync_error++;
+                    break;
                 }
+            } else {
+                LocalResource *dbres = ucx_map_cstr_remove(db->resources, local->path);
+                //local_resource_free(dbres);
             }
         }
-        
-        ucx_list_free(cols);
-    }
-    ucx_map_free_content(db->resources, (ucx_destructor)local_resource_free);
-    ucx_map_free(db->resources);
-    db->resources = lclres;
+        deletelist = cols;
+        col_list = NULL;
+    }   
     
     // unlock repository
     if(locked) {
@@ -1283,15 +1309,10 @@
         remove(locktokenfile);
     }
     
+    //ucx_map_free_content(db->resources, (ucx_destructor)local_resource_free);
+    //ucx_map_free(db->resources);
+    
     dav_session_destroy(sn);
-    while(resources) {
-        UcxList *next = resources->next;
-        if(resources->data) {
-            local_resource_free(resources->data);
-        }
-        free(resources);
-        resources = next;
-    }
     
     // Report
     if(ret != -2) {