# HG changeset patch # User Olaf Wintermann # Date 1509197117 -7200 # Node ID 5f80c5d0e87fc1b6bf9aad5ab735e48ab71f4bae # Parent 000cdd1241150b701acbd2d2b6c9ed1d1b66f8d1 adds tar import/export diff -r 000cdd124115 -r 5f80c5d0e87f dav/Makefile --- a/dav/Makefile Wed Oct 25 17:52:35 2017 +0200 +++ b/dav/Makefile Sat Oct 28 15:25:17 2017 +0200 @@ -35,6 +35,7 @@ DAV_SRC += optparser.c DAV_SRC += error.c DAV_SRC += assistant.c +DAV_SRC += tar.c SYNC_SRC = sync.c SYNC_SRC += config.c diff -r 000cdd124115 -r 5f80c5d0e87f dav/main.c --- a/dav/main.c Wed Oct 25 17:52:35 2017 +0200 +++ b/dav/main.c Sat Oct 28 15:25:17 2017 +0200 @@ -157,8 +157,8 @@ static char *cmdusageinfo[] = { "list [-altdepcR] [-u ] ", - "get [-pcRK] [-o ] [-u ] ", - "put [-pcR] [-k ] [-L ] ", + "get [-pcRKA] [-o ] [-u ] ", + "put [-pcRA] [-k ] [-L ] ", "mkdir [-pc] [-k ] [-L ] ", "remove [-pc] [-L ] ", "copy [-pcO] [-L ] ", @@ -218,6 +218,7 @@ fprintf(stderr, " -d order by last modified date\n"); fprintf(stderr, " -e show extended flags\n"); fprintf(stderr, " -O override resources\n"); + fprintf(stderr, " -A tar import/export\n"); fprintf(stderr, " -L specificy lock token\n"); fprintf(stderr, " -T timeout in seconds\n"); fprintf(stderr, " -n specify namespace uri\n"); @@ -717,12 +718,19 @@ * use stdout if the output file is - */ char *outfile = cmd_getoption(a, "output"); + char *basepath = outfile; + char *tar = cmd_getoption(a, "tar"); if(!outfile) { if(res->iscollection) { - outfile = ""; + basepath = ""; } else { - outfile = res->name; + basepath = res->name; } + if(tar) { + outfile = "-"; + } + } else if(tar) { + basepath = ""; } else if(res->iscollection && !strcmp(outfile, "-")) { fprintf( stderr, @@ -738,7 +746,7 @@ GetResource *getres = malloc(sizeof(GetResource)); getres->res = res; - getres->path = strdup(outfile); + getres->path = strdup(basepath); char *structure = cmd_getoption(a, "structure"); @@ -756,7 +764,7 @@ GetResource *newres = malloc(sizeof(GetResource)); newres->res = child; newres->path = pathlen > 0 ? - util_concat_path(g->path, child->name) : child->name; + util_concat_path(g->path, child->name) : strdup(child->name); stack = ucx_list_prepend(stack, newres); @@ -779,15 +787,36 @@ } } + // download resources int ret; + getfunc get; + TarOutputStream *tout = NULL; + if(tar) { + get = (getfunc)resource2tar; + FILE *tarfile = strcmp(outfile, "-") ? fopen(outfile, "wb") : stdout; + if(!tarfile) { + perror("Cannot open tar output file"); + return -1; + } + tout = tar_open(tarfile); + } else { + get = get_resource; + } UCX_FOREACH(elm, reslist) { GetResource *getres = elm->data; - ret = get_resource(repo, getres->res, a, getres->path); + ret = get(repo, getres, a, tout); if(ret) { break; } } + if(tar) { + // close tar stream + if(tar_close(tout)) { + fprintf(stderr, "tar stream broken\n"); + ret = -1; + } + } ucx_list_free_content(reslist, free_getres); ucx_list_free(reslist); @@ -796,8 +825,9 @@ return ret; } -int get_resource(Repository *repo, DavResource *res, CmdArgs *a, char *out) { - size_t outlen = strlen(out); +int get_resource(Repository *repo, GetResource *getres, CmdArgs *a, void *unused) { + DavResource *res = getres->res; + char *out = res->path; if(res->iscollection) { printf("get: %s\n", res->path); @@ -851,6 +881,35 @@ return 0; } +#define DEFAULT_DIR_MODE T_IRUSR | T_IWUSR | T_IXUSR | T_IRGRP | T_IXGRP | T_IROTH | T_IXOTH +#define DEFAULT_FILE_MODE T_IRUSR | T_IWUSR | T_IRGRP | T_IROTH + +int resource2tar(Repository *repo, GetResource *res, CmdArgs *a, TarOutputStream *tar) { + DavResource *d = res->res; + + if(d->iscollection) { + fprintf(stderr, "add d: %s\n", res->path); + return tar_add_dir(tar, res->path, DEFAULT_DIR_MODE, d->lastmodified); + } + + fprintf(stderr, "add f: %s\n", res->path); + + // add tar file header + if(tar_begin_file(tar, res->path, DEFAULT_FILE_MODE, d->contentlength, d->lastmodified)) { + fprintf(stderr, "TAR Error: %s\n", tar_error2str(tar->error)); + return -1; + } + + if(dav_get_content(d, tar, (dav_write_func)tar_fwrite)) { + print_resource_error(d->session, d->path); + return -1; + } + + // download content + + return tar_end_file(tar); +} + int cmd_put(CmdArgs *a) { if(a->argc != 2) { // TODO: change, when put supports multiple files (however it should do) @@ -883,12 +942,17 @@ } } + char *tar = cmd_getoption(a, "tar"); int ret; - if(!strcmp(file, "-")) { - FILE *in = stdin; - ret = put_file(repo, a, sn, path, "stdin", in, 0); + if(!tar) { + if(!strcmp(file, "-")) { + FILE *in = stdin; + ret = put_file(repo, a, sn, path, "stdin", in, 0); + } else { + ret = put_entry(repo, a, sn, path, file, TRUE); + } } else { - ret = put_entry(repo, a, sn, path, file, TRUE); + ret = put_tar(repo, a, sn, file, path); } free(path); @@ -977,6 +1041,91 @@ return ret; } +int put_tar(Repository *repo, CmdArgs *a, DavSession *sn, char *tarfile, char *path) { + int isstdin = !strcmp(tarfile, "-"); + FILE *in = isstdin ? stdin : fopen(tarfile, "rb"); + if(!in) { + perror("Cannot open tar file"); + return -1; + } + + DavResource *col = NULL; + for(int i=0;i<2;i++) { + col = dav_query(sn, "select - from %s", path); + if(!col && sn->error == DAV_UNAUTHORIZED) { + if(request_auth(repo, sn, a)) { + continue; + } + } + break; + } + if(!col) { + if(sn->error == DAV_NOT_FOUND) { + col = dav_resource_new(sn, path); + col->iscollection = TRUE; + if(dav_create(col)) { + print_resource_error(sn, path); + return -1; + } + } else { + print_resource_error(sn, path); + return -1; + } + } else if(!col->iscollection) { + fprintf(stderr, "%s is not a collection\n", col->href); + return -1; + } + + + int ret = 0; + TarInputStream *tar = tar_inputstream_open(in); + TarEntry *e = NULL; + while((e = tar_read_entry(tar)) != NULL) { + char *newpath = util_concat_path(path, e->path); + if(e->type == TAR_TYPE_FILE) { + fprintf(stderr, "put: %s\n", e->path); + DavResource *res = dav_resource_new(sn, newpath); + dav_set_content(res, tar, (dav_read_func)tar_fread); + dav_set_content_length(res, (size_t)e->size); + + if(dav_store(res)) { + print_resource_error(sn, res->path); + fprintf(stderr, "Cannot upload file.\n"); + if(sn->errorstr) { + fprintf(stderr, "%s\n", sn->errorstr); + } + return -1; + } + + } else if(e->type == TAR_TYPE_DIRECTORY) { + printf("mkcol: %s\n", e->path); + DavResource *res = dav_resource_new(sn, newpath); + res->iscollection = TRUE; + if(!dav_exists(res)) { + if(dav_create(res)) { + fprintf(stderr, "Cannot create collection %s\n", newpath); + print_resource_error(sn, res->path); + ret = 1; + free(newpath); + break; + } + } + } else { + fprintf(stderr, "skip: %s\n", e->path); + } + free(newpath); + } + if(tar->error != TAR_OK) { + ret = -1; + } + + if(!isstdin) { + fclose(in); + } + + return ret; +} + int put_file(Repository *repo, CmdArgs *a, DavSession *sn, char *path, char *name, FILE *in, off_t len) { DavResource *res = NULL; for(int i=0;i<2;i++) { diff -r 000cdd124115 -r 5f80c5d0e87f dav/main.h --- a/dav/main.h Wed Oct 25 17:52:35 2017 +0200 +++ b/dav/main.h Sat Oct 28 15:25:17 2017 +0200 @@ -33,6 +33,7 @@ #include "optparser.h" #include +#include "tar.h" #include "version.h" #ifdef __cplusplus @@ -43,6 +44,8 @@ DavResource *res; char *path; } GetResource; + +typedef int(*getfunc)(Repository *, GetResource *, CmdArgs *, void *); void print_usage(char *cmd); char* password_input(char *prompt); @@ -53,10 +56,12 @@ void ls_print_elm(DavResource *res, char *parent, CmdArgs *args); int cmd_get(CmdArgs *args); -int get_resource(Repository *repo, DavResource *res, CmdArgs *a, char *out); +int get_resource(Repository *repo, GetResource *res, CmdArgs *a, void *unused); +int resource2tar(Repository *repo, GetResource *res, CmdArgs *a, TarOutputStream *tar); int cmd_put(CmdArgs *args); int put_entry(Repository *repo, CmdArgs *a, DavSession *sn, char *path, char *file, DavBool root); +int put_tar(Repository *repo, CmdArgs *a, DavSession *sn, char *tarfile, char *path); int put_file(Repository *repo, CmdArgs *a, DavSession *sn, char *path, char *name, FILE *in, off_t len); int cmd_remove(CmdArgs *args); diff -r 000cdd124115 -r 5f80c5d0e87f dav/optparser.c --- a/dav/optparser.c Wed Oct 25 17:52:35 2017 +0200 +++ b/dav/optparser.c Sat Oct 28 15:25:17 2017 +0200 @@ -128,6 +128,10 @@ ucx_map_cstr_put(a->options, "structure", NOARG); break; } + case 'A': { + ucx_map_cstr_put(a->options, "tar", NOARG); + break; + } case 'K': { ucx_map_cstr_put(a->options, "keep", NOARG); break; diff -r 000cdd124115 -r 5f80c5d0e87f dav/tar.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/dav/tar.c Sat Oct 28 15:25:17 2017 +0200 @@ -0,0 +1,319 @@ +/* + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. + * + * Copyright 2017 Olaf Wintermann. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "tar.h" + +#include +#include +#include + + +const char* tar_error2str(TarError error) { + switch(error) { + case TAR_OK: return "ok"; + case TAR_PATH_TOO_LONG: return "path too long"; + case TAR_FILE_TOO_LARGE: return "file too large"; + case TAR_CONTENT_TOO_LARGE: return "tar content too large"; + case TAR_UNFINISHED_FILE: return "can't read a tar header at this position"; + case TAR_CONTENT_BROKEN: return "tar content broken"; + } + return "error"; +} + +TarOutputStream* tar_open(FILE *f) { + TarOutputStream *tar = malloc(sizeof(TarOutputStream)); + tar->file= f; + tar->cur_filesize = 0; + tar->cur_written = 0; + tar->error = 0; + return tar; +} + + +static int add_header(TarOutputStream *tar, char *path, uint32_t mode, uint64_t size, time_t mtime, int type) { + // split path in prefix and name and check length + char *p = util_parent_path(path); + char *n = util_resource_name(path); + if(!p || !n) { + return -1; + } + + sstr_t prefix = sstr(p); + sstr_t name = sstr(n); + + if(prefix.ptr[prefix.length-1] == '/') { + prefix.length--; + } + + if(prefix.length > 154) { + tar->error = TAR_PATH_TOO_LONG; + return -1; + } + if(name.length > 99) { + tar->error = TAR_PATH_TOO_LONG; + return -1; + } + + // check file length + if(size >= 077777777777 ) { + tar->error = TAR_FILE_TOO_LARGE; + return -1; + } + + // set header fields + TarHeader h; + memset(&h, 0, sizeof(TarHeader)); + + // name + memcpy(h.name, name.ptr, name.length); + // mode + snprintf(h.mode, 8, "%0.7o", mode); + h.mode[7] = ' '; + // uid/gid + memset(h.uid, '0', 16); + h.uid[7] = ' '; + h.gid[7] = ' '; + // size + snprintf(h.size, 12, "%0.11lo", size); + h.size[11] = ' '; + // mtime + uint64_t t = (uint64_t)mtime; + snprintf(h.mtime, 12, "%0.11lo", mtime); + h.mtime[11] = ' '; + // chksum + memset(h.chksum, ' ', 8); + // typeflag + h.typeflag = type; + // linkname - zeros + // magic + snprintf(h.magic, 6, "ustar"); + // version + h.version[0] = '0'; + h.version[1] = '0'; + // uname/gname - zero + // devmajor/devminor + snprintf(h.devmajor, 16, "%0.15o", 0); + h.devmajor[7] = ' '; + h.devminor[7] = ' '; + // prefix + memcpy(h.prefix, prefix.ptr, prefix.length); + + // compute checksum + uint8_t *header = (uint8_t*)&h; + uint32_t chksum = 0; + for(int i=0;i<512;i++) { + chksum += header[i]; + } + snprintf(h.chksum, 8, "%0.7o", chksum); + + fwrite(&h, 1, 512, tar->file); + + return 0; +} + + +int tar_add_dir(TarOutputStream *tar, char *path, uint32_t mode, time_t mtime) { + return add_header(tar, path, mode, 0, mtime, TAR_TYPE_DIRECTORY); +} + +int tar_begin_file( + TarOutputStream *tar, + char *path, + uint32_t mode, + uint64_t size, + time_t mtime) +{ + if(add_header(tar, path, mode, size, mtime, TAR_TYPE_FILE)) { + return -1; + } + + tar->cur_filesize = size; + tar->cur_written = 0; + + return 0; +} + +size_t tar_fwrite(const void *ptr, size_t s, size_t n, TarOutputStream *stream) { + size_t w = fwrite(ptr, s, n, stream->file); + stream->cur_written += w; + if(stream->cur_written > stream->cur_filesize) { + stream->error = TAR_CONTENT_TOO_LARGE; + return 0; + } + return w; +} + +int tar_end_file(TarOutputStream *tar) { + size_t pad = 512 - tar->cur_written % 512; + char buf[512]; + memset(buf, 0, 512); + fwrite(buf, 1, pad, tar->file); + + tar->cur_filesize = 0; + tar->cur_written = 0; + return 0; +} + +int tar_close(TarOutputStream *tar) { + char buf[512]; + memset(buf, 0, 512); + fwrite(buf, 1, 512, tar->file); + return 0; +} + +TarInputStream* tar_inputstream_open(FILE *f) { + TarInputStream *tar = malloc(sizeof(TarInputStream)); + memset(tar, 0, sizeof(TarInputStream)); + tar->file = f; + return tar; +} + +TarEntry* tar_read_entry(TarInputStream *tar) { + if(tar->cur_read < tar->cur_entry.size) { + tar->error = TAR_UNFINISHED_FILE; + return NULL; + } + + TarHeader h; + memset(&h, 0, sizeof(TarHeader)); + + if(fread(&h, 1, 512, tar->file) != 512) { + tar->error = TAR_ERROR; + return NULL; + } + + // some checks + tar->error = TAR_CONTENT_BROKEN; // set error for all following returns + + char *buf = (char*)&h; + uint64_t chksum = 0; + int chksumfield = 8 * 32; + for(int i=0;i<148;i++) { + chksum += buf[i]; + } + chksum += chksumfield; + for(int i=156;i<512;i++) { + chksum += buf[i]; + } + + if(chksum != chksumfield) { + long long int cks = strtoll(h.chksum, NULL, 8); + if(cks != chksum) { + return NULL; + } + } + + if(memcmp(h.magic, "ustar\0", 6)) { + return NULL; + } + if(memcmp(h.version, "00", 2)) { + return NULL; + } + + // check if name and prefix are null terminated + int nameterm = 0; + int prefixterm = 0; + for(int i=0;i<100;i++) { + if(h.name[i] == 0) { + nameterm = 1; + break; + } + } + for(int i=0;i<155;i++) { + if(h.prefix[i] == 0) { + prefixterm = 1; + break; + } + } + if(!nameterm || !prefixterm) { + return NULL; + } + + // get size + if(h.size[11] != ' ') { + return NULL; + } + long long int size = strtoll(h.size, NULL, 8); + if(size < 0) { + return NULL; + } + + if(h.name[0] == 0) { + return NULL; + } + + // get path + char *path = h.prefix[0] != 0 ? util_concat_path(h.prefix, h.name) : strdup(h.name); + + if(tar->cur_entry.path) { + free(tar->cur_entry.path); + } + + tar->cur_entry.path = path; + tar->cur_entry.size = (uint64_t)size; + tar->cur_entry.type = h.typeflag; + tar->cur_read = 0; + + tar->error = TAR_OK; + return &tar->cur_entry; +} + +size_t tar_fread(void *ptr, size_t s, size_t n, TarInputStream *stream) { + size_t bufsize = s*n; + size_t available = stream->cur_entry.size - stream->cur_read; + + size_t nb = available > bufsize ? bufsize : available; + if(nb == 0) { + return 0; + } + size_t r = fread(ptr, 1, nb, stream->file); + if(r != nb) { + stream->error = TAR_ERROR; + return 0; + } + + stream->cur_read += r; + if(stream->cur_read >= stream->cur_entry.size) { + // read padding + size_t pad = 512 - stream->cur_read % 512; + char buf[512]; + if(fread(buf, 1, pad, stream->file) != pad) { + stream->error = TAR_ERROR; + return 0; + } + } + + return r; +} + +int tar_inputstream_close(TarInputStream *tar) { + if(tar->cur_entry.path) { + free(tar->cur_entry.path); + } + free(tar); +} diff -r 000cdd124115 -r 5f80c5d0e87f dav/tar.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/dav/tar.h Sat Oct 28 15:25:17 2017 +0200 @@ -0,0 +1,133 @@ +/* + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. + * + * Copyright 2017 Olaf Wintermann. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef TAR_H +#define TAR_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define T_ISUID 04000 +#define T_ISGID 02000 +#define T_IRUSR 00400 +#define T_IWUSR 00200 +#define T_IXUSR 00100 +#define T_IRGRP 00040 +#define T_IWGRP 00020 +#define T_IXGRP 00010 +#define T_IROTH 00004 +#define T_IWOTH 00002 +#define T_IXOTH 00001 + +#define TAR_TYPE_FILE '0' +#define TAR_TYPE_DIRECTORY '5' + +enum TarError { + TAR_OK = 0, + TAR_PATH_TOO_LONG, + TAR_FILE_TOO_LARGE, + TAR_CONTENT_TOO_LARGE, + TAR_UNFINISHED_FILE, + TAR_CONTENT_BROKEN, + TAR_ERROR +}; + +typedef enum TarError TarError; + +typedef struct TarHeader { + char name[100]; + char mode[8]; + char uid[8]; + char gid[8]; + char size[12]; + char mtime[12]; + char chksum[8]; + char typeflag; + char linkname[100]; + char magic[6]; + char version[2]; + char uname[32]; + char gname[32]; + char devmajor[8]; + char devminor[8]; + char prefix[155]; + char padding[12]; +} TarHeader; + +typedef struct TarOutputStream { + FILE *file; + uint64_t cur_filesize; + uint64_t cur_written; + TarError error; +} TarOutputStream; + +typedef struct TarEntry { + char *path; + uint64_t size; + int type; +} TarEntry; + +typedef struct TarInputStream { + FILE *file; + TarEntry cur_entry; + uint64_t cur_read; + TarError error; +} TarInputStream; + +const char* tar_error2str(TarError error); + +TarOutputStream* tar_open(FILE *f); +int tar_add_dir(TarOutputStream *tar, char *path, uint32_t mode, time_t mtime); +int tar_begin_file( + TarOutputStream *tar, + char *path, + uint32_t mode, + uint64_t size, + time_t mtime); +size_t tar_fwrite(const void *ptr, size_t s, size_t n, TarOutputStream *stream); +int tar_end_file(TarOutputStream *tar); +int tar_close(TarOutputStream *tar); + +TarInputStream* tar_inputstream_open(FILE *f); +TarEntry* tar_read_entry(TarInputStream *tar); +size_t tar_fread(void *ptr, size_t s, size_t n, TarInputStream *stream); +int tar_inputstream_close(TarInputStream *tar); + + +#ifdef __cplusplus +} +#endif + +#endif /* TAR_H */ +