# HG changeset patch # User Olaf Wintermann # Date 1574247465 -3600 # Node ID 3320429502cfa35dd7d4f7ff53376d896651906b # Parent e66f2645be65013f2fcb8f466950da15e03376f5 fix some split bugs and add tests diff -r e66f2645be65 -r 3320429502cf dav/db.c --- a/dav/db.c Wed Nov 20 08:44:26 2019 +0100 +++ b/dav/db.c Wed Nov 20 11:57:45 2019 +0100 @@ -739,6 +739,21 @@ return s ? strdup(s) : NULL; } +void local_resource_copy_parts(LocalResource *from, LocalResource *to) { + if(from->parts) { + to->numparts = from->numparts; + to->parts = calloc(from->numparts, sizeof(FilePart)); + for(int i=0;inumparts;i++) { + FilePart s = from->parts[i]; + FilePart p; + p.block = s.block; + p.hash = nullstrdup(s.hash); + p.etag = nullstrdup(s.etag); + to->parts[i] = p; + } + } +} + LocalResource* local_resource_copy(LocalResource *src, const char *new_path) { LocalResource *newres = calloc(1, sizeof(LocalResource)); newres->path = strdup(new_path); @@ -769,18 +784,7 @@ newres->xattr_hash = nullstrdup(src->xattr_hash); newres->remote_tags_hash = nullstrdup(src->remote_tags_hash); - if(src->parts) { - newres->numparts = src->numparts; - newres->parts = calloc(src->numparts, sizeof(FilePart)); - for(int i=0;inumparts;i++) { - FilePart s = src->parts[i]; - FilePart p; - p.block = s.block; - p.hash = nullstrdup(s.hash); - p.etag = nullstrdup(s.etag); - newres->parts[i] = p; - } - } + local_resource_copy_parts(src, newres); newres->blocksize = src->blocksize; diff -r e66f2645be65 -r 3320429502cf dav/db.h --- a/dav/db.h Wed Nov 20 08:44:26 2019 +0100 +++ b/dav/db.h Wed Nov 20 11:57:45 2019 +0100 @@ -110,6 +110,8 @@ void local_resource_free(LocalResource *res); +void local_resource_copy_parts(LocalResource *from, LocalResource *to); + LocalResource* local_resource_copy(LocalResource *src, const char *new_path); void filepart_free(FilePart *part); diff -r e66f2645be65 -r 3320429502cf dav/sync.c --- a/dav/sync.c Wed Nov 20 08:44:26 2019 +0100 +++ b/dav/sync.c Wed Nov 20 11:57:45 2019 +0100 @@ -143,7 +143,7 @@ // ignore sigpipe to make sure the program doesn't exit // if stdout will be closed (for example by using dav-sync ... | head) struct sigaction act; - ZERO(&act, sizeof(struct sigaction)); + memset(&act, 0, sizeof(struct sigaction)); act.sa_handler = SIG_IGN; sigaction(SIGPIPE, &act, NULL); @@ -578,7 +578,7 @@ } int ret = 0; - DavResource *ls = dav_query(sn, "select D:getetag,idav:status,idav:split,`idav:content-hash`,idavprops:tags,idavprops:finfo,idavprops:xattributes,idavprops:link from / with depth = infinity"); + DavResource *ls = dav_query(sn, "select D:getetag,idav:split,idav:status,`idav:content-hash`,idavprops:tags,idavprops:finfo,idavprops:xattributes,idavprops:link from / with depth = infinity"); if(!ls) { print_resource_error(sn, "/"); if(locked) { @@ -975,7 +975,7 @@ return REMOTE_NO_CHANGE; } char *hash = sync_get_content_hash(res); - + DavBool issplit = dav_get_property(res, "idav:split") ? TRUE : FALSE; if(issplit) { util_remove_trailing_pathseparator(res->path); @@ -1017,6 +1017,11 @@ if(SYNC_SYMLINK(dir) && nullstrcmp(link, local->link_target)) { ret = REMOTE_CHANGE_LINK; nochange = TRUE; + } else if(issplit && local->hash && hash) { + if(!strcmp(local->hash, hash)) { + // resource is already up-to-date on the client + nochange = TRUE; + } } else if(local->etag) { sstr_t e = sstr(etag); if(sstrprefix(e, S("W/"))) { @@ -1545,7 +1550,9 @@ } // set metadata from stat - local_resource_set_etag(local, etag); + if(!issplit) { + local_resource_set_etag(local, etag); + } if(content_hash) { local->hash = content_hash; } @@ -2822,21 +2829,7 @@ { LocalResource *db_res = ucx_map_cstr_get(db->resources, res->path); res->tags_updated = 0; - if(db_res) { - if(svrres) { - DavResource *remote = ucx_map_cstr_get(svrres, res->path); - if(restore_removed && !remote) { - return 1; - } - if(!res->isdirectory && restore_modified && remote) { - char *etag = dav_get_string_property(remote, "D:getetag"); - if(!etag || (db_res->etag && strcmp(etag, db_res->etag))) { - res->restore = TRUE; - return 1; - } - } - } - + if(db_res) { // copy some metadata from db_res, that localscan does not deliver res->tags_updated = db_res->tags_updated; if(db_res->etag) { @@ -2855,13 +2848,25 @@ res->prev_hash = strdup(db_res->hash); } - // if the resource is splitted, move the part infos to the new + // if the resource is splitted, copy the part info to the new // LocalResource obj, because we need it later if(db_res->parts) { - res->parts = db_res->parts; - res->numparts = db_res->numparts; - db_res->parts = NULL; - db_res->numparts = 0; + local_resource_copy_parts(db_res, res); + } + + // check if the file must be restored on the server + if(svrres) { + DavResource *remote = ucx_map_cstr_get(svrres, res->path); + if(restore_removed && !remote) { + return 1; + } + if(!res->isdirectory && restore_modified && remote) { + char *etag = dav_get_string_property(remote, "D:getetag"); + if(!etag || (db_res->etag && strcmp(etag, db_res->etag))) { + res->restore = TRUE; + return 1; + } + } } // check if metadata has changed @@ -2996,7 +3001,7 @@ return 0; } - if(remote->iscollection) { + if(remote->iscollection && !res->parts) { return 1; } @@ -3116,7 +3121,7 @@ } } - if(local_blocksize > 0 && svr_blocksize > 0) { + if(local_blocksize > 0 && svr_blocksize > 0 && local_blocksize != svr_blocksize) { fprintf(stderr, "Warning: Blocksize mismatch: %s: local: %zu server: %zu\n", local->path, local_blocksize, svr_blocksize); return svr_blocksize; } else if(local_blocksize > 0) { diff -r e66f2645be65 -r 3320429502cf dav/version.h --- a/dav/version.h Wed Nov 20 08:44:26 2019 +0100 +++ b/dav/version.h Wed Nov 20 11:57:45 2019 +0100 @@ -27,6 +27,6 @@ */ #ifndef DAV_VERSION -#define DAV_VERSION "1.3 beta1" +#define DAV_VERSION "1.3 beta2" #endif diff -r e66f2645be65 -r 3320429502cf test/bin-test/dav-home/sync.xml --- a/test/bin-test/dav-home/sync.xml Wed Nov 20 08:44:26 2019 +0100 +++ b/test/bin-test/dav-home/sync.xml Wed Nov 20 11:57:45 2019 +0100 @@ -212,4 +212,43 @@ + + + + test6a + $HOME/tmp-sync/test6a + dav-test-repo + /sync/test6 + .trash + dav-sync-tests-test6a-db.xml + + true + + + + 4m + 1m + + + + + + test6b + $HOME/tmp-sync/test6b + dav-test-repo + /sync/test6 + .trash + dav-sync-tests-test6b-db.xml + + true + + + + 4m + 1m + + + diff -r e66f2645be65 -r 3320429502cf test/bin-test/test-dav-sync-split1.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/bin-test/test-dav-sync-split1.sh Wed Nov 20 11:57:45 2019 +0100 @@ -0,0 +1,241 @@ +#!/bin/sh +# +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. +# +# Copyright 2019 Olaf Wintermann. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# + +if [ -z "$DAV_BIN" ]; +then + echo "DAV_BIN variable not set" + exit 1 +fi +if [ -z "$DAV_SYNC_BIN" ]; +then + echo "DAV_BIN variable not set" + exit 1 +fi + +XATTR=../../build/xattrtool + +# checks if tmp-sync/out.txt contains a specific text +# arg1: pattern +# arg2: errormsg +check_tmpout() +{ + TEST=`cat tmp-sync/out.txt | grep "$1"` + if [ $? -ne 0 ]; + then + echo "$2" + exit 2 + fi +} + +# checks if tmp-sync/out.txt does not contain a specific text +# arg1: pattern +# arg2: errormsg +ncheck_tmpout() +{ + TEST=`cat tmp-sync/out.txt | grep "$1"` + if [ $? -eq 0 ]; + then + echo "$2" + exit 2 + fi +} + +# do dav-sync push and check return value +# arg1: dir +# arg2: errormsg +dav_sync_push() +{ + $DAV_SYNC_BIN push $1 > tmp-sync/out.txt + if [ $? -ne 0 ]; + then + echo "$2" + exit 2 + fi +} +# do dav-sync pull and check return value +# arg1: dir +# arg2: errormsg +dav_sync_pull() +{ + $DAV_SYNC_BIN pull $1 > tmp-sync/out.txt + if [ $? -ne 0 ]; + then + echo "$2" + exit 2 + fi +} + +rm -f .dav/dav-sync-tests-test6a-db.xml +rm -f .dav/dav-sync-tests-test6b-db.xml + +$DAV_BIN rm dav-test-repo/sync/test6 2> /dev/null + +$DAV_BIN mkcol dav-test-repo/sync/test6 2> /dev/null + +# tmp sync dir +rm -Rf tmp-sync +mkdir tmp-sync +mkdir tmp-sync/test6a +mkdir tmp-sync/test6b + +# ---------------------------------------------------------------------------- +# test 1: add some small files and sync +# expected result: everything synced, no split + +mkdir tmp-sync/test6a/dir1 +mkdir tmp-sync/test6a/dir1/sub1 + +cp synctest/file1 tmp-sync/test6a +cp synctest/file2 tmp-sync/test6a/dir1 +cp synctest/file3 tmp-sync/test6a/dir1/sub1 + +dav_sync_push test6a "test 1: push failed" +check_tmpout "3 files pushed" "test 1: wrong push counter" +check_tmpout "0 conflicts" "test 1: wrong conflict counter (push)" +check_tmpout "0 errors" "test 1: wrong error counter (push)" + +dav_sync_pull test6b "test 1: pull failed" +check_tmpout "3 files pulled" "test 1: wrong pull counter" +check_tmpout "0 conflicts" "test 1: wrong conflict counter (pull)" +check_tmpout "0 errors" "test 1: wrong error counter (pull)" + +$DAV_BIN info dav-test-repo/sync/test6/file1 | grep "type: resource" > /dev/null 2>&1 +if [ $? -ne 0 ]; then + echo "test 1: file1 not a resource" + exit 2 +fi +$DAV_BIN info dav-test-repo/sync/test6/dir1/file2 | grep "type: resource" > /dev/null 2>&1 +if [ $? -ne 0 ]; then + echo "test 1: file2 not a resource" + exit 2 +fi +$DAV_BIN info dav-test-repo/sync/test6/dir1/sub1/file3 | grep "type: resource" > /dev/null 2>&1 +if [ $? -ne 0 ]; then + echo "test 1: file3 not a resource" + exit 2 +fi + +# ---------------------------------------------------------------------------- +# test 2: add a large file +# expected result: split + +head -c 1048576 /dev/zero | sed 's/\x0/a/g' > tmp-sync/xa +cat tmp-sync/xa | sed 's/a/b/g' > tmp-sync/xb +cat tmp-sync/xa | sed 's/a/c/g' > tmp-sync/xc +cat tmp-sync/xa | sed 's/a/1/g' > tmp-sync/x1 +cat tmp-sync/xa | sed 's/a/2/g' > tmp-sync/x2 +cat tmp-sync/xa | sed 's/a/3/g' > tmp-sync/x3 + +cat tmp-sync/xa tmp-sync/xb tmp-sync/xc tmp-sync/x1 tmp-sync/x2 tmp-sync/x3 > tmp-sync/test6a/big1 + +dav_sync_push test6a "test 2: push failed" +check_tmpout "1 file pushed" "test 2: wrong push counter" +check_tmpout "0 conflicts" "test 2: wrong conflict counter (push)" +check_tmpout "0 errors" "test 2: wrong error counter (push)" + +dav_sync_pull test6b "test 2: pull failed" +check_tmpout "1 file pulled" "test 2: wrong pull counter" +check_tmpout "0 conflicts" "test 2: wrong conflict counter (pull)" +check_tmpout "0 errors" "test 2: wrong error counter (pull)" + +$DAV_BIN info dav-test-repo/sync/test6/big1 | grep "type: collection" > /dev/null 2>&1 +if [ $? -ne 0 ]; then + echo "test 2: big1 not a collection" + exit 2 +fi + +HASH1=`cat tmp-sync/test6a/big1 | sha256sum` +HASH2=`cat tmp-sync/test6b/big1 | sha256sum` +if [ "$HASH1" != "$HASH2" ]; then + echo "test 2: big1 not equal" + exit 2 +fi + +$DAV_BIN list -l dav-test-repo/sync/test6/big1 > tmp-sync/list.txt 2> /dev/null +if [ $? -ne 0 ]; then + echo "test 2: dav list failed" + exit 2 +fi + +TEST=`cat tmp-sync/list.txt | wc -l` +if [ $TEST -ne 6 ]; then + echo "test 2: wrong block count" + exit 2 +fi + +LASTMODIFIED_PART0=`$DAV_BIN get-property dav-test-repo/sync/test6/big1/0 D:getlastmodified` +LASTMODIFIED_PART1=`$DAV_BIN get-property dav-test-repo/sync/test6/big1/1 D:getlastmodified` +LASTMODIFIED_PART2=`$DAV_BIN get-property dav-test-repo/sync/test6/big1/2 D:getlastmodified` +LASTMODIFIED_PART3=`$DAV_BIN get-property dav-test-repo/sync/test6/big1/3 D:getlastmodified` +LASTMODIFIED_PART4=`$DAV_BIN get-property dav-test-repo/sync/test6/big1/4 D:getlastmodified` +LASTMODIFIED_PART5=`$DAV_BIN get-property dav-test-repo/sync/test6/big1/5 D:getlastmodified` + + +# ---------------------------------------------------------------------------- +# test 3: modify first block +# expected result: first block synced + +sleep 2 + +dd if=tmp-sync/xc of=tmp-sync/test6a/big1 conv=notrunc > /dev/null 2>&1 + +dav_sync_push test6a "test 3: push failed" +check_tmpout "1 file pushed" "test 3: wrong push counter" +check_tmpout "0 conflicts" "test 3: wrong conflict counter (push)" +check_tmpout "0 errors" "test 3: wrong error counter (push)" + +dav_sync_pull test6b "test 3: pull failed" +check_tmpout "1 file pulled" "test 3: wrong pull counter" +check_tmpout "0 conflicts" "test 3: wrong conflict counter (pull)" +check_tmpout "0 errors" "test 3: wrong error counter (pull)" + +HASH1=`cat tmp-sync/test6a/big1 | sha256sum` +HASH2=`cat tmp-sync/test6b/big1 | sha256sum` +if [ "$HASH1" != "$HASH2" ]; then + echo "test 3: big1 not equal" + exit 2 +fi + +LASTMODIFIED_PART0_2=`$DAV_BIN get-property dav-test-repo/sync/test6/big1/0 D:getlastmodified` +LASTMODIFIED_PART1_2=`$DAV_BIN get-property dav-test-repo/sync/test6/big1/1 D:getlastmodified` +LASTMODIFIED_PART2_2=`$DAV_BIN get-property dav-test-repo/sync/test6/big1/2 D:getlastmodified` + +if [ "$LASTMODIFIED_PART0" = "$LASTMODIFIED_PART0_2" ]; then + echo "test 3: part 0 not updated" + exit 2 +fi +if [ "$LASTMODIFIED_PART1" != "$LASTMODIFIED_PART1_2" ]; then + echo "test 3: part 1 updated" + exit 2 +fi +if [ "$LASTMODIFIED_PART2" != "$LASTMODIFIED_PART2_2" ]; then + echo "test 3: part 2 updated" + exit 2 +fi + diff -r e66f2645be65 -r 3320429502cf test/bin-test/test-dav-sync.sh --- a/test/bin-test/test-dav-sync.sh Wed Nov 20 08:44:26 2019 +0100 +++ b/test/bin-test/test-dav-sync.sh Wed Nov 20 11:57:45 2019 +0100 @@ -106,6 +106,7 @@ do_test "dav-sync metadata (3)" test-dav-sync-metadata3.sh do_test "dav-sync metadata (4)" test-dav-sync-metadata4.sh do_test "dav-sync versioning (1)" test-dav-sync-versioning1.sh +do_test "dav-sync split (1)" test-dav-sync-split1.sh # cleanup $DAV_BIN rm dav-test-repo/sync/test1 > /dev/null 2>&1