Mercurial > hg > dav / changeset

--- a/dav/db.c	Wed Nov 20 08:44:26 2019 +0100
+++ b/dav/db.c	Wed Nov 20 11:57:45 2019 +0100
@@ -739,6 +739,21 @@
     return s ? strdup(s) : NULL;
 }
 
+void local_resource_copy_parts(LocalResource *from, LocalResource *to) {
+    if(from->parts) {
+        to->numparts = from->numparts;
+        to->parts = calloc(from->numparts, sizeof(FilePart));
+        for(int i=0;i<to->numparts;i++) {
+            FilePart s = from->parts[i];
+            FilePart p;
+            p.block = s.block;
+            p.hash = nullstrdup(s.hash);
+            p.etag = nullstrdup(s.etag);
+            to->parts[i] = p;
+        }
+    }
+}
+
 LocalResource* local_resource_copy(LocalResource *src, const char *new_path) {
     LocalResource *newres = calloc(1, sizeof(LocalResource));
     newres->path = strdup(new_path);
@@ -769,18 +784,7 @@
     newres->xattr_hash = nullstrdup(src->xattr_hash);
     newres->remote_tags_hash = nullstrdup(src->remote_tags_hash);
     
-    if(src->parts) {
-        newres->numparts = src->numparts;
-        newres->parts = calloc(src->numparts, sizeof(FilePart));
-        for(int i=0;i<newres->numparts;i++) {
-            FilePart s = src->parts[i];
-            FilePart p;
-            p.block = s.block;
-            p.hash = nullstrdup(s.hash);
-            p.etag = nullstrdup(s.etag);
-            newres->parts[i] = p;
-        }
-    }
+    local_resource_copy_parts(src, newres);
     
     newres->blocksize = src->blocksize;
     

--- a/dav/db.h	Wed Nov 20 08:44:26 2019 +0100
+++ b/dav/db.h	Wed Nov 20 11:57:45 2019 +0100
@@ -110,6 +110,8 @@
 
 void local_resource_free(LocalResource *res);
 
+void local_resource_copy_parts(LocalResource *from, LocalResource *to);
+
 LocalResource* local_resource_copy(LocalResource *src, const char *new_path);
 
 void filepart_free(FilePart *part);

--- a/dav/sync.c	Wed Nov 20 08:44:26 2019 +0100
+++ b/dav/sync.c	Wed Nov 20 11:57:45 2019 +0100
@@ -143,7 +143,7 @@
     // ignore sigpipe to make sure the program doesn't exit
     // if stdout will be closed (for example by using dav-sync ... | head)
     struct sigaction act;
-    ZERO(&act, sizeof(struct sigaction));
+    memset(&act, 0, sizeof(struct sigaction));
     act.sa_handler = SIG_IGN;
     sigaction(SIGPIPE, &act, NULL);
     
@@ -578,7 +578,7 @@
     }
     
     int ret = 0;
-    DavResource *ls = dav_query(sn, "select D:getetag,idav:status,idav:split,`idav:content-hash`,idavprops:tags,idavprops:finfo,idavprops:xattributes,idavprops:link from / with depth = infinity");
+    DavResource *ls = dav_query(sn, "select D:getetag,idav:split,idav:status,`idav:content-hash`,idavprops:tags,idavprops:finfo,idavprops:xattributes,idavprops:link from / with depth = infinity");
     if(!ls) {
         print_resource_error(sn, "/");
         if(locked) {
@@ -975,7 +975,7 @@
         return REMOTE_NO_CHANGE;
     }
     char *hash = sync_get_content_hash(res);
-    
+      
     DavBool issplit = dav_get_property(res, "idav:split") ? TRUE : FALSE;
     if(issplit) {
         util_remove_trailing_pathseparator(res->path);
@@ -1017,6 +1017,11 @@
         if(SYNC_SYMLINK(dir) && nullstrcmp(link, local->link_target)) {
             ret = REMOTE_CHANGE_LINK;
             nochange = TRUE;
+        } else if(issplit && local->hash && hash) {
+            if(!strcmp(local->hash, hash)) {
+                // resource is already up-to-date on the client
+                nochange = TRUE;
+            }
         } else if(local->etag) {
             sstr_t e = sstr(etag);
             if(sstrprefix(e, S("W/"))) {
@@ -1545,7 +1550,9 @@
         }
         
         // set metadata from stat
-        local_resource_set_etag(local, etag);
+        if(!issplit) {
+            local_resource_set_etag(local, etag);
+        }
         if(content_hash) {
             local->hash = content_hash;
         }
@@ -2822,21 +2829,7 @@
 {
     LocalResource *db_res = ucx_map_cstr_get(db->resources, res->path);
     res->tags_updated = 0;
-    if(db_res) {
-        if(svrres) {
-            DavResource *remote = ucx_map_cstr_get(svrres, res->path);
-            if(restore_removed && !remote) {
-                return 1;
-            }
-            if(!res->isdirectory && restore_modified && remote) {
-                char *etag = dav_get_string_property(remote, "D:getetag");
-                if(!etag || (db_res->etag && strcmp(etag, db_res->etag))) {
-                    res->restore = TRUE;
-                    return 1;
-                }
-            }
-        }
-        
+    if(db_res) { 
         // copy some metadata from db_res, that localscan does not deliver
         res->tags_updated = db_res->tags_updated;
         if(db_res->etag) {
@@ -2855,13 +2848,25 @@
             res->prev_hash = strdup(db_res->hash);
         }
         
-        // if the resource is splitted, move the part infos to the new
+        // if the resource is splitted, copy the part info to the new
         // LocalResource obj, because we need it later
         if(db_res->parts) {
-            res->parts = db_res->parts;
-            res->numparts = db_res->numparts;
-            db_res->parts = NULL;
-            db_res->numparts = 0;
+            local_resource_copy_parts(db_res, res);
+        }
+        
+        // check if the file must be restored on the server
+        if(svrres) {
+            DavResource *remote = ucx_map_cstr_get(svrres, res->path);
+            if(restore_removed && !remote) {
+                return 1;
+            }
+            if(!res->isdirectory && restore_modified && remote) {
+                char *etag = dav_get_string_property(remote, "D:getetag");
+                if(!etag || (db_res->etag && strcmp(etag, db_res->etag))) {
+                    res->restore = TRUE;
+                    return 1;
+                }
+            }
         }
         
         // check if metadata has changed
@@ -2996,7 +3001,7 @@
         return 0;
     }
     
-    if(remote->iscollection) {
+    if(remote->iscollection && !res->parts) {
         return 1;
     }
     
@@ -3116,7 +3121,7 @@
         }
     }
     
-    if(local_blocksize > 0 && svr_blocksize > 0) {
+    if(local_blocksize > 0 && svr_blocksize > 0 && local_blocksize != svr_blocksize) {
         fprintf(stderr, "Warning: Blocksize mismatch: %s: local: %zu server: %zu\n", local->path, local_blocksize, svr_blocksize);
         return svr_blocksize;
     } else if(local_blocksize > 0) {

--- a/dav/version.h	Wed Nov 20 08:44:26 2019 +0100
+++ b/dav/version.h	Wed Nov 20 11:57:45 2019 +0100
@@ -27,6 +27,6 @@
  */
 
 #ifndef DAV_VERSION
-#define DAV_VERSION "1.3 beta1"
+#define DAV_VERSION "1.3 beta2"
 #endif
 

--- a/test/bin-test/dav-home/sync.xml	Wed Nov 20 08:44:26 2019 +0100
+++ b/test/bin-test/dav-home/sync.xml	Wed Nov 20 11:57:45 2019 +0100
@@ -212,4 +212,43 @@
 		
 		<versioning type="simple" always="true" />
 	</directory>
+	
+	<!--
+	Test 6: split
+	-->
+	<directory>
+		<name>test6a</name>
+		<path>$HOME/tmp-sync/test6a</path>
+		<repository>dav-test-repo</repository>
+		<collection>/sync/test6</collection>
+		<trash>.trash</trash>
+		<database>dav-sync-tests-test6a-db.xml</database>
+		
+		<hashing>true</hashing>
+		
+		<splitconfig>
+			<split>
+				<minsize>4m</minsize>
+				<blocksize>1m</blocksize>
+			</split>
+		</splitconfig>
+	</directory>
+	
+	<directory>
+		<name>test6b</name>
+		<path>$HOME/tmp-sync/test6b</path>
+		<repository>dav-test-repo</repository>
+		<collection>/sync/test6</collection>
+		<trash>.trash</trash>
+		<database>dav-sync-tests-test6b-db.xml</database>
+		
+		<hashing>true</hashing>
+		
+		<splitconfig>
+			<split>
+				<minsize>4m</minsize>
+				<blocksize>1m</blocksize>
+			</split>
+		</splitconfig>
+	</directory>
 </configuration>

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/bin-test/test-dav-sync-split1.sh	Wed Nov 20 11:57:45 2019 +0100
@@ -0,0 +1,241 @@
+#!/bin/sh
+#
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+#
+# Copyright 2019 Olaf Wintermann. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#   1. Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in the
+#      documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+
+if [ -z "$DAV_BIN" ];
+then
+	echo "DAV_BIN variable not set"
+	exit 1
+fi
+if [ -z "$DAV_SYNC_BIN" ];
+then
+	echo "DAV_BIN variable not set"
+	exit 1
+fi
+
+XATTR=../../build/xattrtool
+
+# checks if tmp-sync/out.txt contains a specific text
+# arg1: pattern
+# arg2: errormsg
+check_tmpout()
+{
+	TEST=`cat tmp-sync/out.txt | grep "$1"`
+	if [ $? -ne 0 ];
+	then
+		echo "$2"
+		exit 2	
+	fi
+}
+
+# checks if tmp-sync/out.txt does not contain a specific text
+# arg1: pattern
+# arg2: errormsg
+ncheck_tmpout()
+{
+	TEST=`cat tmp-sync/out.txt | grep "$1"`
+	if [ $? -eq 0 ];
+	then
+		echo "$2"
+		exit 2	
+	fi
+}
+
+# do dav-sync push and check return value
+# arg1: dir
+# arg2: errormsg
+dav_sync_push()
+{
+	$DAV_SYNC_BIN push $1 > tmp-sync/out.txt
+	if [ $? -ne 0 ];
+	then
+		echo "$2"
+		exit 2
+	fi
+}
+# do dav-sync pull and check return value
+# arg1: dir
+# arg2: errormsg
+dav_sync_pull()
+{
+	$DAV_SYNC_BIN pull $1 > tmp-sync/out.txt
+	if [ $? -ne 0 ];
+	then
+		echo "$2"
+		exit 2
+	fi
+}
+
+rm -f .dav/dav-sync-tests-test6a-db.xml
+rm -f .dav/dav-sync-tests-test6b-db.xml
+
+$DAV_BIN rm dav-test-repo/sync/test6 2> /dev/null
+
+$DAV_BIN mkcol dav-test-repo/sync/test6 2> /dev/null
+
+# tmp sync dir
+rm -Rf tmp-sync
+mkdir tmp-sync
+mkdir tmp-sync/test6a
+mkdir tmp-sync/test6b
+
+# ----------------------------------------------------------------------------
+# test 1: add some small files and sync
+# expected result: everything synced, no split
+
+mkdir tmp-sync/test6a/dir1
+mkdir tmp-sync/test6a/dir1/sub1
+
+cp synctest/file1 tmp-sync/test6a
+cp synctest/file2 tmp-sync/test6a/dir1
+cp synctest/file3 tmp-sync/test6a/dir1/sub1
+
+dav_sync_push test6a "test 1: push failed"
+check_tmpout "3 files pushed" "test 1: wrong push counter"
+check_tmpout "0 conflicts" "test 1: wrong conflict counter (push)"
+check_tmpout "0 errors" "test 1: wrong error counter (push)"
+
+dav_sync_pull test6b "test 1: pull failed"
+check_tmpout "3 files pulled" "test 1: wrong pull counter"
+check_tmpout "0 conflicts" "test 1: wrong conflict counter (pull)"
+check_tmpout "0 errors" "test 1: wrong error counter (pull)"
+
+$DAV_BIN info dav-test-repo/sync/test6/file1 | grep "type: resource" > /dev/null 2>&1
+if [ $? -ne 0 ]; then
+	echo "test 1: file1 not a resource"
+	exit 2
+fi
+$DAV_BIN info dav-test-repo/sync/test6/dir1/file2 | grep "type: resource" > /dev/null 2>&1
+if [ $? -ne 0 ]; then
+	echo "test 1: file2 not a resource"
+	exit 2
+fi
+$DAV_BIN info dav-test-repo/sync/test6/dir1/sub1/file3 | grep "type: resource" > /dev/null 2>&1
+if [ $? -ne 0 ]; then
+	echo "test 1: file3 not a resource"
+	exit 2
+fi
+
+# ----------------------------------------------------------------------------
+# test 2: add a large file
+# expected result: split
+
+head -c 1048576 /dev/zero | sed 's/\x0/a/g' > tmp-sync/xa
+cat tmp-sync/xa | sed 's/a/b/g' > tmp-sync/xb
+cat tmp-sync/xa | sed 's/a/c/g' > tmp-sync/xc
+cat tmp-sync/xa | sed 's/a/1/g' > tmp-sync/x1
+cat tmp-sync/xa | sed 's/a/2/g' > tmp-sync/x2
+cat tmp-sync/xa | sed 's/a/3/g' > tmp-sync/x3
+
+cat tmp-sync/xa tmp-sync/xb tmp-sync/xc tmp-sync/x1 tmp-sync/x2 tmp-sync/x3 > tmp-sync/test6a/big1
+
+dav_sync_push test6a "test 2: push failed"
+check_tmpout "1 file pushed" "test 2: wrong push counter"
+check_tmpout "0 conflicts" "test 2: wrong conflict counter (push)"
+check_tmpout "0 errors" "test 2: wrong error counter (push)"
+
+dav_sync_pull test6b "test 2: pull failed"
+check_tmpout "1 file pulled" "test 2: wrong pull counter"
+check_tmpout "0 conflicts" "test 2: wrong conflict counter (pull)"
+check_tmpout "0 errors" "test 2: wrong error counter (pull)"
+
+$DAV_BIN info dav-test-repo/sync/test6/big1 | grep "type: collection" > /dev/null 2>&1
+if [ $? -ne 0 ]; then
+	echo "test 2: big1 not a collection"
+	exit 2
+fi
+
+HASH1=`cat tmp-sync/test6a/big1 | sha256sum`
+HASH2=`cat tmp-sync/test6b/big1 | sha256sum`
+if [ "$HASH1" != "$HASH2" ]; then
+	echo "test 2: big1 not equal"
+	exit 2
+fi
+
+$DAV_BIN list -l dav-test-repo/sync/test6/big1 > tmp-sync/list.txt 2> /dev/null
+if [ $? -ne 0 ]; then
+	echo "test 2: dav list failed"
+	exit 2
+fi
+
+TEST=`cat tmp-sync/list.txt | wc -l`
+if [ $TEST -ne 6 ]; then
+	echo "test 2: wrong block count"
+	exit 2
+fi
+
+LASTMODIFIED_PART0=`$DAV_BIN get-property dav-test-repo/sync/test6/big1/0 D:getlastmodified`
+LASTMODIFIED_PART1=`$DAV_BIN get-property dav-test-repo/sync/test6/big1/1 D:getlastmodified`
+LASTMODIFIED_PART2=`$DAV_BIN get-property dav-test-repo/sync/test6/big1/2 D:getlastmodified`
+LASTMODIFIED_PART3=`$DAV_BIN get-property dav-test-repo/sync/test6/big1/3 D:getlastmodified`
+LASTMODIFIED_PART4=`$DAV_BIN get-property dav-test-repo/sync/test6/big1/4 D:getlastmodified`
+LASTMODIFIED_PART5=`$DAV_BIN get-property dav-test-repo/sync/test6/big1/5 D:getlastmodified`
+
+
+# ----------------------------------------------------------------------------
+# test 3: modify first block
+# expected result: first block synced
+
+sleep 2
+
+dd if=tmp-sync/xc of=tmp-sync/test6a/big1 conv=notrunc > /dev/null 2>&1
+
+dav_sync_push test6a "test 3: push failed"
+check_tmpout "1 file pushed" "test 3: wrong push counter"
+check_tmpout "0 conflicts" "test 3: wrong conflict counter (push)"
+check_tmpout "0 errors" "test 3: wrong error counter (push)"
+
+dav_sync_pull test6b "test 3: pull failed"
+check_tmpout "1 file pulled" "test 3: wrong pull counter"
+check_tmpout "0 conflicts" "test 3: wrong conflict counter (pull)"
+check_tmpout "0 errors" "test 3: wrong error counter (pull)"
+
+HASH1=`cat tmp-sync/test6a/big1 | sha256sum`
+HASH2=`cat tmp-sync/test6b/big1 | sha256sum`
+if [ "$HASH1" != "$HASH2" ]; then
+	echo "test 3: big1 not equal"
+	exit 2
+fi
+
+LASTMODIFIED_PART0_2=`$DAV_BIN get-property dav-test-repo/sync/test6/big1/0 D:getlastmodified`
+LASTMODIFIED_PART1_2=`$DAV_BIN get-property dav-test-repo/sync/test6/big1/1 D:getlastmodified`
+LASTMODIFIED_PART2_2=`$DAV_BIN get-property dav-test-repo/sync/test6/big1/2 D:getlastmodified`
+
+if [ "$LASTMODIFIED_PART0" = "$LASTMODIFIED_PART0_2" ]; then
+	echo "test 3: part 0 not updated"
+	exit 2
+fi
+if [ "$LASTMODIFIED_PART1" != "$LASTMODIFIED_PART1_2" ]; then
+	echo "test 3: part 1 updated"
+	exit 2
+fi
+if [ "$LASTMODIFIED_PART2" != "$LASTMODIFIED_PART2_2" ]; then
+	echo "test 3: part 2 updated"
+	exit 2
+fi
+

--- a/test/bin-test/test-dav-sync.sh	Wed Nov 20 08:44:26 2019 +0100
+++ b/test/bin-test/test-dav-sync.sh	Wed Nov 20 11:57:45 2019 +0100
@@ -106,6 +106,7 @@
 do_test "dav-sync metadata (3)" test-dav-sync-metadata3.sh
 do_test "dav-sync metadata (4)" test-dav-sync-metadata4.sh
 do_test "dav-sync versioning (1)" test-dav-sync-versioning1.sh
+do_test "dav-sync split (1)" test-dav-sync-split1.sh
 
 # cleanup
 $DAV_BIN rm dav-test-repo/sync/test1 > /dev/null 2>&1