changeset
fix net_http_write when used with chunked transfer encoding and non-blocking IO
Sun, 04 Jun 2023 20:09:18 +0200
- author
- Olaf Wintermann <olaf.wintermann@gmail.com>
- date
- Sun, 04 Jun 2023 20:09:18 +0200
- changeset 498
- 0d80f8a2b29f
- parent 497
- 8827517054ec
- child 499
- ef77854a91f3
fix net_http_write when used with chunked transfer encoding and non-blocking IO
--- a/src/server/public/nsapi.h Wed May 31 19:39:10 2023 +0200 +++ b/src/server/public/nsapi.h Sun Jun 04 20:09:18 2023 +0200 @@ -1367,7 +1367,7 @@ #define servact_translate_uri servact_translate_uri #define request_translate_uri servact_translate_uri -ssize_t net_write(SYS_NETFD fd, void *buf, size_t nbytes); +ssize_t net_write(SYS_NETFD fd, const void *buf, size_t nbytes); ssize_t net_writev(SYS_NETFD fd, struct iovec *iovec, int iovcnt); ssize_t net_sendfile(SYS_NETFD fd, sendfiledata *sfd); ssize_t net_read(SYS_NETFD fd, void *buf, size_t nbytes);
--- a/src/server/safs/cgi.c Wed May 31 19:39:10 2023 +0200 +++ b/src/server/safs/cgi.c Sun Jun 04 20:09:18 2023 +0200 @@ -179,7 +179,7 @@ Event *writeev = pool_malloc(sn->pool, sizeof(Event)); ZERO(writeev, sizeof(Event)); writeev->cookie = handler; - // TODO: fn + handler->writeev = writeev; handler->stderrev = stderr_readev; @@ -256,12 +256,13 @@ if(!handler->poll_out) { if(event_pollout(ev, sn->csd, handler->writeev)) { handler->result = REQ_ABORTED; - return 1; + return 0; } handler->poll_out = TRUE; } } else { handler->result = REQ_ABORTED; + log_ereport(LOG_FAILURE, "cgi_try_write: %s", strerror(errno)); } return 1; } @@ -292,8 +293,12 @@ // try to flush handler->writebuf // if writebuf is empty, this does nothing and returns 0 if(cgi_try_write_flush(handler, sn)) { - log_ereport(LOG_DEBUG, "cgi-send: req: %p write failed: abort", rq); - return handler->result == REQ_ABORTED ? 0 : 1; + if(handler->result == REQ_ABORTED) { + log_ereport(LOG_DEBUG, "cgi-send: req: %p write failed: abort", rq); + return 0; + } else { + return 1; + } } char buf[4096]; // I/O buffer @@ -462,9 +467,12 @@ Request *rq = parser->rq; log_ereport(LOG_DEBUG, "cgi-send: req: %p event-finish", rq); - if(handler->result == REQ_ABORTED) { + if(handler->result == REQ_ABORTED && handler->process.pid != 0) { log_ereport(LOG_FAILURE, "cgi-send: kill script: %s", handler->path); - kill(handler->process.pid, SIGKILL); + + killpg(handler->process.pid, SIGTERM); + + handler->process.pid = 0; } if(!handler->stderr_finished) {
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/server/test/io.c Sun Jun 04 20:09:18 2023 +0200 @@ -0,0 +1,805 @@ +/* + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. + * + * Copyright 2023 Olaf Wintermann. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "io.h" + +#include "testutils.h" + + +UCX_TEST(test_io_http_stream_parse_chunk_header_hdronly_first) { + char *str = strdup("100\r\n"); + size_t len = strlen(str); + char *str2 = strdup("12345\r\n"); + size_t len2 = strlen(str2); + char *str3 = strdup("FF\r\n"); + size_t len3 = strlen(str3); + + UCX_TEST_BEGIN; + + int64_t chunklen; + int ret = http_stream_parse_chunk_header(str, len, TRUE, &chunklen); + UCX_TEST_ASSERT(ret == 5, "ret != 5"); + UCX_TEST_ASSERT(chunklen == 0x100, "wrong chunk length"); + + // test 2 + ret = http_stream_parse_chunk_header(str2, len2, TRUE, &chunklen); + UCX_TEST_ASSERT(ret == 7, "ret != 7 (test 2)"); + UCX_TEST_ASSERT(chunklen == 0x12345, "wrong chunk length (test 2)"); + + // test 3: hex test + ret = http_stream_parse_chunk_header(str3, len3, TRUE, &chunklen); + UCX_TEST_ASSERT(ret == 4, "ret != 7 (test 3)"); + UCX_TEST_ASSERT(chunklen == 0xFF, "wrong chunk length (test 3)"); + + UCX_TEST_END; + free(str); + free(str2); +} + +UCX_TEST(test_io_http_stream_parse_chunk_header_hdronly) { + char *str = strdup("\r\n100\r\n"); + size_t len = strlen(str); + char *str2 = strdup("\nab\n"); + size_t len2 = strlen(str2); + + UCX_TEST_BEGIN; + + int64_t chunklen; + int ret = http_stream_parse_chunk_header(str, len, FALSE, &chunklen); + UCX_TEST_ASSERT(ret == 7, "ret != 7"); + UCX_TEST_ASSERT(chunklen == 0x100, "wrong chunk length"); + + // test 2 with just \n as line break + ret = http_stream_parse_chunk_header(str2, len2, FALSE, &chunklen); + UCX_TEST_ASSERT(ret == 4, "ret != 5 (test 2)"); + UCX_TEST_ASSERT(chunklen == 0xab, "wrong chunk length (test 2)"); + + UCX_TEST_END; + free(str); + free(str2); +} + +UCX_TEST(test_io_http_stream_parse_chunk_header_hdronly_seq_fail) { + // test: after the first chunk header, \r\n is required before any new header + char *str = strdup("ff\r\n"); + size_t len = strlen(str); + + UCX_TEST_BEGIN; + + int64_t chunklen; + int ret = http_stream_parse_chunk_header(str, len, FALSE, &chunklen); + UCX_TEST_ASSERT(ret == -1, "ret != -1"); + + UCX_TEST_END; + free(str); +} + +UCX_TEST(test_io_http_stream_parse_chunk_header_hdr_data) { + char *str = strdup("\r\nb\r\nhello world\r\n"); + size_t len = strlen(str); + + UCX_TEST_BEGIN; + + int64_t chunklen; + int ret = http_stream_parse_chunk_header(str, len, FALSE, &chunklen); + UCX_TEST_ASSERT(ret == 5, "ret != 5"); + + UCX_TEST_END; + free(str); +} + +UCX_TEST(test_io_http_stream_parse_chunk_header_empty) { + char *str = ""; + size_t len = strlen(str); + + UCX_TEST_BEGIN; + + int64_t chunklen; + int ret = http_stream_parse_chunk_header(str, len, FALSE, &chunklen); + UCX_TEST_ASSERT(ret == 0, "ret != 0"); + + ret = http_stream_parse_chunk_header(str, len, TRUE, &chunklen); + UCX_TEST_ASSERT(ret == 0, "ret != 0 (test 2)"); + + UCX_TEST_END; +} + +UCX_TEST(test_io_http_stream_parse_chunk_header_partial_first) { + char *str = strdup("123"); + size_t len = strlen(str); + + UCX_TEST_BEGIN; + + int64_t chunklen; + int ret = http_stream_parse_chunk_header(str, len, TRUE, &chunklen); + UCX_TEST_ASSERT(ret == 0, "ret != 0"); + + UCX_TEST_END; + free(str); +} + +UCX_TEST(test_io_http_stream_parse_chunk_header_partial) { + char *str = strdup("123"); + size_t len = strlen(str); + char *str2 = strdup("\r\n"); + size_t len2 = strlen(str2); + char *str3 = strdup("\r"); + size_t len3 = strlen(str3); + char *str4 = strdup("\r\n123"); + size_t len4 = strlen(str4); + + UCX_TEST_BEGIN; + + int64_t chunklen; + int ret = http_stream_parse_chunk_header(str, len, TRUE, &chunklen); + UCX_TEST_ASSERT(ret == 0, "ret != 0"); + ret = http_stream_parse_chunk_header(str2, len2, FALSE, &chunklen); + UCX_TEST_ASSERT(ret == 0, "ret != 0"); + ret = http_stream_parse_chunk_header(str3, len3, FALSE, &chunklen); + UCX_TEST_ASSERT(ret == 0, "ret != 0"); + ret = http_stream_parse_chunk_header(str4, len4, FALSE, &chunklen); + UCX_TEST_ASSERT(ret == 0, "ret != 0"); + + UCX_TEST_END; + free(str); + free(str2); + free(str3); + free(str4); +} + +UCX_TEST(test_io_http_stream_parse_chunk_header_invalid) { + char *str = strdup("hello\r\n"); + size_t len = strlen(str); + char *str2 = strdup("x4\r\n\r\n123\r\n"); + size_t len2 = strlen(str2); + char *str3 = strdup("\r\n\r\n123\r\n"); + size_t len3 = strlen(str3); + + UCX_TEST_BEGIN; + + int64_t chunklen; + int ret; + + ret = http_stream_parse_chunk_header(str, len, TRUE, &chunklen); + UCX_TEST_ASSERT(ret == -1, "ret != -1 (test 1a)"); + ret = http_stream_parse_chunk_header(str, len, FALSE, &chunklen); + UCX_TEST_ASSERT(ret == -1, "ret != -1 (test 1b)"); + + ret = http_stream_parse_chunk_header(str2, len2, TRUE, &chunklen); + UCX_TEST_ASSERT(ret == -1, "ret != -1 (test 1a)"); + ret = http_stream_parse_chunk_header(str2, len2, FALSE, &chunklen); + UCX_TEST_ASSERT(ret == -1, "ret != -1 (test 1b)"); + + ret = http_stream_parse_chunk_header(str3, len3, TRUE, &chunklen); + UCX_TEST_ASSERT(ret == -1, "ret != -1 (test 1a)"); + ret = http_stream_parse_chunk_header(str3, len3, FALSE, &chunklen); + UCX_TEST_ASSERT(ret == -1, "ret != -1 (test 1b)"); + + UCX_TEST_END; + free(str); + free(str2); + free(str3); +} + +UCX_TEST(test_io_http_stream_parse_chunk_header_zero) { + char *str = strdup("\r\n0\r\n\r\n"); + size_t len = strlen(str); + char *str2 = strdup("0\r\n\r\n"); + size_t len2 = strlen(str2); + + // incomplete + char *str3 = strdup("\r\n0\r\n"); + size_t len3 = strlen(str3); + char *str4 = strdup("\r\n0"); + size_t len4 = strlen(str4); + + + UCX_TEST_BEGIN; + + int64_t chunklen = -1; + int ret = http_stream_parse_chunk_header(str, len, FALSE, &chunklen); + UCX_TEST_ASSERT(ret == 7, "ret != 7"); + UCX_TEST_ASSERT(chunklen == 0, "chunklen != 0"); + + chunklen = -1; + ret = http_stream_parse_chunk_header(str2, len2, TRUE, &chunklen); + UCX_TEST_ASSERT(ret == 5, "ret != 5 (test 2)"); + UCX_TEST_ASSERT(chunklen == 0, "chunklen != 0 (test 2)"); + + // expect 0 (incomplete) + ret = http_stream_parse_chunk_header(str3, len3, FALSE, &chunklen); + UCX_TEST_ASSERT(ret == 0, "ret != 3 (test 3)"); + + ret = http_stream_parse_chunk_header(str4, len4, FALSE, &chunklen); + UCX_TEST_ASSERT(ret == 0, "ret != 3 (test 4)"); + + UCX_TEST_END; + free(str); + free(str2); + free(str3); + free(str4); +} + + +UCX_TEST(test_io_httpstream_write) { + Session *sn = testutil_session(); + + TestIOStream *st = testutil_iostream(2048, TRUE); + IOStream *http = httpstream_new(sn->pool, (IOStream*)st); + + UCX_TEST_BEGIN; + + char *msg = "hello world!"; + size_t msglen = strlen(msg); + + ssize_t w = net_write(http, msg, msglen); + + UCX_TEST_ASSERT(w == msglen, "wrong size returned by net_write"); + UCX_TEST_ASSERT(st->buf->size == msglen, "wrong buffer size"); + UCX_TEST_ASSERT(!memcmp(st->buf->space, msg, msglen), "wrong buffer content"); + + // test again, make sure the second message is written directly after the wirst one + char *msg2 = "test"; + size_t msglen2 = strlen(msg2); + + w = net_write(http, msg2, msglen2); + + UCX_TEST_ASSERT(w == msglen2, "wrong size returned by net_write (2)"); + UCX_TEST_ASSERT(st->buf->size == msglen+msglen2, "wrong buffer size (2)"); + UCX_TEST_ASSERT(!memcmp(st->buf->space + msglen, msg2, msglen2), "wrong buffer content (2)"); + + UCX_TEST_END; + + testutil_destroy_session(sn); +} + +UCX_TEST(test_io_httpstream_chunked_write) { + Session *sn = testutil_session(); + + TestIOStream *st = testutil_iostream(2048, TRUE); + IOStream *http = httpstream_new(sn->pool, (IOStream*)st); + httpstream_enable_chunked_write(http); + + UCX_TEST_BEGIN; + + char *msg = "hello world!"; + size_t msglen = strlen(msg); + + char *bufmsg = "c\r\nhello world!\r\n"; + size_t bufmsglen = strlen(bufmsg); + + ssize_t w = net_write(http, msg, msglen); + + cxstring s1 = cx_strn(st->buf->space, st->buf->size); + cxstring s2 = cx_strn(bufmsg, bufmsglen); + + UCX_TEST_ASSERT(w == msglen, "wrong size returned by net_write"); + UCX_TEST_ASSERT(st->buf->size == bufmsglen, "wrong buffer size"); + UCX_TEST_ASSERT(!cx_strcasecmp(s1, s2), "wrong buffer content"); + + // write again + w = net_write(http, msg, msglen); + UCX_TEST_ASSERT(w == msglen, "write 2: wrong return value"); + UCX_TEST_ASSERT(st->buf->size == 2*bufmsglen, "write 2: wrong buf size"); + + cxstring s3 = cx_strn(st->buf->space+bufmsglen, bufmsglen); + UCX_TEST_ASSERT(!cx_strcasecmp(s2, s3), "write 2: wrong buf content"); + + + UCX_TEST_END; +} + +UCX_TEST(test_io_httpstream_chunked_write_end) { + Session *sn = testutil_session(); + + TestIOStream *st = testutil_iostream(2048, TRUE); + IOStream *http = httpstream_new(sn->pool, (IOStream*)st); + httpstream_enable_chunked_write(http); + + UCX_TEST_BEGIN; + + char *msg = "hello world!"; + size_t msglen = strlen(msg); + + char *bufmsg = "c\r\nhello world!\r\n0\r\n\r\n"; + size_t bufmsglen = strlen(bufmsg); + + ssize_t w = net_write(http, msg, msglen); + net_finish(http); + + cxstring s1 = cx_strn(st->buf->space, st->buf->size); + cxstring s2 = cx_strn(bufmsg, bufmsglen); + + UCX_TEST_ASSERT(w == msglen, "wrong size returned by net_write"); + UCX_TEST_ASSERT(st->buf->size == bufmsglen, "wrong buffer size"); + UCX_TEST_ASSERT(!cx_strcasecmp(s1, s2), "wrong buffer content"); + + + UCX_TEST_END; +} + +UCX_TEST(test_io_httpstream_chunked_write_xx) { + // This test creates a giant buffer and writes it with + // chunked transfer encoding to the http stream with varying chunk length + + Session *sn = testutil_session(); + + TestIOStream *st = testutil_iostream(2048, TRUE); + IOStream *http = httpstream_new(sn->pool, (IOStream*)st); + httpstream_enable_chunked_write(http); + + UCX_TEST_BEGIN; + + // create test data + CxBuffer *testdata = cxBufferCreate(NULL, 1024*1024*4, cxDefaultAllocator, 0); + for(size_t i=0;i<testdata->capacity;i++) { + cxBufferPut(testdata, 35+(i%91)); + } + + // write chunks, start with single diget chunk length and increase + // chunk size with each step + size_t pos = 0; + int j = 0; + ssize_t i=15; + while(pos<testdata->size) { + char *buf = testdata->space + pos; + size_t remaining = testdata->size - pos; + ssize_t len = pos + i < remaining ? i : remaining; + pos += len; + + ssize_t w = net_write(http, buf, len); + + UCX_TEST_ASSERT(w == len, "wrong size returned by net_write"); + i+=100; // increase chunk size + j++; // debug + } + + // terminate chunk + net_finish(http); + + // code below also used in test_io_httpstream_chunked_write_xx_limit + + // make sure the output is correctly encoded + // extract chunks from st->buf by using http_stream_parse_chunk_header + // (which should be well-tested) + WSBool first_chunk = TRUE; + int64_t chunklen = 0; + + char *buf = st->buf->space; + size_t bufsize = st->buf->size; + + pos = 0; // st->buf position + size_t srcpos = 0; // testdata position + int debug_counter = 0; + while(pos < bufsize) { + ssize_t remaining = bufsize - pos; + ssize_t src_remaining = testdata->size - srcpos; + + int ret = http_stream_parse_chunk_header(buf+pos, remaining, first_chunk, &chunklen); + first_chunk = FALSE; + + // ret must always be > 0 (0: incomplete chunk header, -1: invalid syntax) + UCX_TEST_ASSERT(ret > 0, "http_stream_parse_chunk_header ret <= 0"); + if(chunklen == 0) { + UCX_TEST_ASSERT(src_remaining == 0, "stream end reached but src_remaining > 0"); + break; + } + + UCX_TEST_ASSERT(chunklen <= src_remaining, "chunklen > src_remaining"); + + char *src_chunk = testdata->space+srcpos; + char *buf_chunk = buf+pos+ret; + + UCX_TEST_ASSERT(!memcmp(buf_chunk, src_chunk, chunklen), "memcmp failed"); + + pos += ret + chunklen; + srcpos += chunklen; + + debug_counter++; + } + + cxBufferFree(testdata); + testutil_destroy_session(sn); + testutil_iostream_destroy(st); + + UCX_TEST_END; +} + +UCX_TEST(test_io_httpstream_chunked_write_partial_header) { + Session *sn = testutil_session(); + + TestIOStream *st = testutil_iostream(2048, TRUE); + IOStream *http = httpstream_new(sn->pool, (IOStream*)st); + httpstream_enable_chunked_write(http); + + UCX_TEST_BEGIN; + + memset(st->buf->space, 0, st->buf->capacity); + + char *msg = "hello world!"; + size_t msglen = strlen(msg); + + st->max_write = 1; // limit the test stream max write size + + // only 1 byte of the header is written, 0 bytes of msg + ssize_t w = net_write(http, msg, msglen); + UCX_TEST_ASSERT(w == 0, "write 1: wrong return value"); + UCX_TEST_ASSERT(st->buf->size == 1, "write 1: wrong buf size"); + UCX_TEST_ASSERT(tolower(st->buf->space[0]) == 'c', "write 1: wrong buf content"); + + // next header byte: '\r' + w = net_write(http, msg, msglen); + UCX_TEST_ASSERT(w == 0, "write 2: wrong return value"); + UCX_TEST_ASSERT(st->buf->size == 2, "write 2: wrong buf size"); + UCX_TEST_ASSERT(st->buf->space[1] == '\r', "write 2: wrong content"); + + // next header byte: '\n' + w = net_write(http, msg, msglen); + UCX_TEST_ASSERT(w == 0, "write 3: wrong return value"); + UCX_TEST_ASSERT(st->buf->size == 3, "write 3: wrong buf size"); + UCX_TEST_ASSERT(st->buf->space[2] == '\n', "write 3: wrong content"); + + // next: content + w = net_write(http, msg, msglen); + UCX_TEST_ASSERT(w == 1, "write 4: wrong return value"); + UCX_TEST_ASSERT(st->buf->size == 4, "write 3: wrong buf size"); + UCX_TEST_ASSERT(st->buf->space[3] == msg[0], "write 3: wrong content"); + + testutil_destroy_session(sn); + testutil_iostream_destroy(st); + + UCX_TEST_END; +} + +UCX_TEST(test_io_httpstream_chunked_write_partial_data) { + Session *sn = testutil_session(); + + TestIOStream *st = testutil_iostream(2048, TRUE); + IOStream *http = httpstream_new(sn->pool, (IOStream*)st); + httpstream_enable_chunked_write(http); + + UCX_TEST_BEGIN; + + memset(st->buf->space, 0, st->buf->capacity); + + char *msg = "hello world!"; + size_t msglen = strlen(msg); + size_t msglen_orig = msglen; + + // limit first write to 3 to only write the header + st->max_write = 3; + + ssize_t w = net_write(http, msg, msglen); + + UCX_TEST_ASSERT(w == 0, "write 1: wrong return value"); + UCX_TEST_ASSERT(st->buf->size == 3, "write 1: wrong buf size"); + UCX_TEST_ASSERT(st->buf->space[0] == 'c', "write 1: wrong buf content"); + UCX_TEST_ASSERT(st->buf->space[2] == '\n', "write 1: wrong buf content"); + + w = net_write(http, msg, msglen); + UCX_TEST_ASSERT(w == 3, "write 2: wrong return value"); + UCX_TEST_ASSERT(st->buf->size == 6, "write 2: wrong buf size"); + UCX_TEST_ASSERT(!memcmp(st->buf->space, "c\r\nhel\0", 7), "write 2: wrong buf content"); + + msg += w; + msglen -= w; + + w = net_write(http, msg, msglen); + UCX_TEST_ASSERT(w == 3, "write 3: wrong return value"); + UCX_TEST_ASSERT(st->buf->size == 9, "write 3: wrong buf size"); + UCX_TEST_ASSERT(!memcmp(st->buf->space, "c\r\nhello \0", 10), "write 3: wrong buf content"); + + st->max_write = 1024; + msg += w; + msglen -= w; + + w = net_write(http, msg, msglen); + UCX_TEST_ASSERT(w == msglen, "write 4: wrong return value"); + UCX_TEST_ASSERT(st->buf->size == 3 + msglen_orig + 2, "write 4: wrong buf size"); + UCX_TEST_ASSERT(!memcmp(st->buf->space, "c\r\nhello world!\r\n", st->buf->size), "write 4: wrong buf content"); + + testutil_destroy_session(sn); + testutil_iostream_destroy(st); + + UCX_TEST_END; +} + +UCX_TEST(test_io_httpstream_chunked_write_partial_trailer) { + Session *sn = testutil_session(); + + TestIOStream *st = testutil_iostream(2048, TRUE); + IOStream *http = httpstream_new(sn->pool, (IOStream*)st); + httpstream_enable_chunked_write(http); + + UCX_TEST_BEGIN; + + memset(st->buf->space, 0, st->buf->capacity); + + char *msg = "hello world!"; + size_t msglen = strlen(msg); + + char *msg2 = "newmsg"; + size_t msglen2 = strlen(msg2); + + char *msg3 = "msg3"; + size_t msglen3 = strlen(msg3); + + st->max_write = 3 + msglen; // header + msg, but without trailer + + ssize_t w = net_write(http, msg, msglen); + + UCX_TEST_ASSERT(w == msglen, "write 1: wrong return value"); + UCX_TEST_ASSERT(st->buf->size == 3 + msglen, "write 1: wrong buf size"); + UCX_TEST_ASSERT(!memcmp(st->buf->space, "c\r\nhello world!\0", st->buf->size + 1), "write 1: wrong buf content"); + + st->max_write = 2 + 3 + msglen2; // trailer + new header + new msg, without new trailer + + w = net_write(http, msg2, msglen2); + UCX_TEST_ASSERT(w == msglen2, "write 2: wrong return value"); + UCX_TEST_ASSERT(st->buf->size == 3 + msglen + 2 + 3 + msglen2, "write 2: wrong buf size"); + UCX_TEST_ASSERT(!memcmp(st->buf->space, "c\r\nhello world!\r\n6\r\nnewmsg\0", st->buf->size + 1), "write 2: wrong buf content"); + + // limit write to 1 byte: two writes required for trailer, net_write should return 0 + st->max_write = 1; + + w = net_write(http, "dummymsg", 8); + UCX_TEST_ASSERT(w == 0, "write 3: wrong return value"); + + w = net_write(http, "dummymsg", 8); + UCX_TEST_ASSERT(w == 0, "write 4: wrong return value"); + UCX_TEST_ASSERT(st->buf->size == 3 + msglen + 2 + 3 + msglen2 + 2, "write 4: wrong buf size"); + UCX_TEST_ASSERT(!memcmp(st->buf->space, "c\r\nhello world!\r\n6\r\nnewmsg\r\n\0", st->buf->size + 1), "write 4: wrong buf content"); + + st->max_write = 1024; + w = net_write(http, msg3, msglen3); + + UCX_TEST_ASSERT(w == msglen3, "write 5: wrong return value"); + UCX_TEST_ASSERT(st->buf->size == 3 + msglen + 2 + 3 + msglen2 + 2 + 3 + msglen3 + 2, "write 5: wrong buf size"); + UCX_TEST_ASSERT(!memcmp(st->buf->space, "c\r\nhello world!\r\n6\r\nnewmsg\r\n4\r\nmsg3\r\n", st->buf->size + 1), "write 5: wrong buf content"); + + + testutil_destroy_session(sn); + testutil_iostream_destroy(st); + + UCX_TEST_END; +} + +UCX_TEST(test_io_httpstream_chunked_write_partial_trailer_partial_header) { + Session *sn = testutil_session(); + + TestIOStream *st = testutil_iostream(2048, TRUE); + IOStream *http = httpstream_new(sn->pool, (IOStream*)st); + httpstream_enable_chunked_write(http); + + UCX_TEST_BEGIN; + + memset(st->buf->space, 0, st->buf->capacity); + + char *msg = "hello world!"; + size_t msglen = strlen(msg); + + char *msg2 = "newmsg"; + size_t msglen2 = strlen(msg2); + + // Test: write partial trailer followed by partial header write + + st->max_write = 3 + msglen + 1; + + ssize_t w = net_write(http, msg, msglen); + + UCX_TEST_ASSERT(w == msglen, "write 1: wrong return value"); + UCX_TEST_ASSERT(st->buf->size == 3 + msglen + 1, "write 1: wrong buf size"); + UCX_TEST_ASSERT(!memcmp(st->buf->space, "c\r\nhello world!\r\0", st->buf->size + 1), "write 1: wrong buf content"); + + st->max_write = 2; // write 1 trailer byte and 1 header byte + + w = net_write(http, msg2, msglen2); + + UCX_TEST_ASSERT(w == 0, "write 2: wrong return value"); + UCX_TEST_ASSERT(st->buf->size == 3 + msglen + 2 + 1, "write 2: wrong buf size"); + UCX_TEST_ASSERT(!memcmp(st->buf->space, "c\r\nhello world!\r\n6\0", st->buf->size + 1), "write 2: wrong buf content"); + + // force partial header write again + st->max_write = 1; + + w = net_write(http, msg2, msglen2); + + UCX_TEST_ASSERT(w == 0, "write 3: wrong return value"); + UCX_TEST_ASSERT(st->buf->size == 3 + msglen + 2 + 2, "write 3: wrong buf size"); + UCX_TEST_ASSERT(!memcmp(st->buf->space, "c\r\nhello world!\r\n6\r\0", st->buf->size + 1), "write 3: wrong buf content"); + + st->max_write = 1024; + + w = net_write(http, msg2, msglen2); + + UCX_TEST_ASSERT(w ==msglen2, "write 4: wrong return value"); + UCX_TEST_ASSERT(st->buf->size == 3 + msglen + 2 + 3 + msglen2 + 2, "write 4: wrong buf size"); + UCX_TEST_ASSERT(!memcmp(st->buf->space, "c\r\nhello world!\r\n6\r\nnewmsg\r\n", st->buf->size + 1), "write 4: wrong buf content"); + + + testutil_destroy_session(sn); + testutil_iostream_destroy(st); + + UCX_TEST_END; +} + +UCX_TEST(test_io_httpstream_chunked_write_data_2x) { + Session *sn = testutil_session(); + + TestIOStream *st = testutil_iostream(2048, TRUE); + IOStream *http = httpstream_new(sn->pool, (IOStream*)st); + httpstream_enable_chunked_write(http); + + UCX_TEST_BEGIN; + + memset(st->buf->space, 0, st->buf->capacity); + + // Test: First write a partial header, which forces a chunk with a specific + // size. After that, write a message, that is bigger than the first + // chunk, forcing a start of a second chunk, in one big writev op. + + char *msg = "hello world!"; + size_t msglen = strlen(msg); + + char *msg2 = "newmsg"; + size_t msglen2 = strlen(msg2); + + char *msg_big = "hello world!newmsg"; + size_t msglen_big = strlen(msg_big); + + st->max_write = 1; + + ssize_t w = net_write(http, msg, msglen); // first chunk: msg + + UCX_TEST_ASSERT(w == 0, "write 1: wrong return value"); + UCX_TEST_ASSERT(st->buf->size == 1, "write 1: wrong buf size"); + + st->max_write = 1024; + + w = net_write(http, msg_big, msglen_big); // first chunk + new chunk + + UCX_TEST_ASSERT(w == msglen_big, "write 2: wrong return value"); + UCX_TEST_ASSERT(st->buf->size == 3 + msglen + 2 + 3 + msglen2 + 2, "write 2: wrong buf size"); + UCX_TEST_ASSERT(!memcmp(st->buf->space, "c\r\nhello world!\r\n6\r\nnewmsg\r\n", st->buf->size + 1), "write 2: wrong buf content"); + + + testutil_destroy_session(sn); + testutil_iostream_destroy(st); + + UCX_TEST_END; +} + +UCX_TEST(test_io_httpstream_chunked_write_xx_limit) { + Session *sn = testutil_session(); + + TestIOStream *st = testutil_iostream(2048, TRUE); + IOStream *http = httpstream_new(sn->pool, (IOStream*)st); + httpstream_enable_chunked_write(http); + + UCX_TEST_BEGIN; + + // Test: create testdata and write it in varying chunk sizes, but + // limit TestIOStream to 1 to 3 byte writes + + // create test data + CxBuffer *testdata = cxBufferCreate(NULL, 1024*16, cxDefaultAllocator, 0); + for(size_t i=0;i<testdata->capacity;i++) { + cxBufferPut(testdata, 35+(i%91)); + } + + st->max_write = 1; + + size_t pos = 0; + int chunksize = 1; + while(pos < testdata->size) { + size_t available = testdata->size - pos; + + char *chunk = testdata->space + pos; + size_t chunklen = chunksize > available ? available : chunksize; + + // write chunk + size_t chunkpos = 0; + int max_writes = chunklen + 24; // max number of write attempts + int writes = 0; + while(chunkpos < chunklen) { + ssize_t w = net_write(http, chunk+chunkpos, chunklen-chunkpos); + UCX_TEST_ASSERT(w >= 0, "net_write failed"); + chunkpos += w; + + writes++; + UCX_TEST_ASSERT(writes < max_writes, "max writes attempts reached"); + } + + pos += chunklen; + chunksize += 5; + + // increase max write size at some point + if(pos + chunksize >= testdata->size) { + st->max_write = INT_MAX; + } else if(pos > 1024*2) { + if(pos < 1024*8) { + st->max_write = 2; + } else { + st->max_write = 3; + } + } + } + + // terminate chunk + net_finish(http); + + + // same code as test_io_httpstream_chunked_write_xx + + // make sure the output is correctly encoded + // extract chunks from st->buf by using http_stream_parse_chunk_header + // (which should be well-tested) + + WSBool first_chunk = TRUE; + int64_t chunklen = 0; + + char *buf = st->buf->space; + size_t bufsize = st->buf->size; + + pos = 0; // st->buf position + size_t srcpos = 0; // testdata position + int debug_counter = 0; + while(pos < bufsize) { + ssize_t remaining = bufsize - pos; + ssize_t src_remaining = testdata->size - srcpos; + + int ret = http_stream_parse_chunk_header(buf+pos, remaining, first_chunk, &chunklen); + first_chunk = FALSE; + + // ret must always be > 0 (0: incomplete chunk header, -1: invalid syntax) + UCX_TEST_ASSERT(ret > 0, "http_stream_parse_chunk_header ret <= 0"); + if(chunklen == 0) { + UCX_TEST_ASSERT(src_remaining == 0, "stream end reached but src_remaining > 0"); + break; + } + + UCX_TEST_ASSERT(chunklen <= src_remaining, "chunklen > src_remaining"); + + char *src_chunk = testdata->space+srcpos; + char *buf_chunk = buf+pos+ret; + + UCX_TEST_ASSERT(!memcmp(buf_chunk, src_chunk, chunklen), "memcmp failed"); + + pos += ret + chunklen; + srcpos += chunklen; + + debug_counter++; + } + + + testutil_destroy_session(sn); + testutil_iostream_destroy(st); + cxBufferFree(testdata); + + UCX_TEST_END; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/server/test/io.h Sun Jun 04 20:09:18 2023 +0200 @@ -0,0 +1,69 @@ +/* + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. + * + * Copyright 2023 Olaf Wintermann. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef TEST_IO_H +#define TEST_IO_H + +#include "../public/nsapi.h" + +#include "test.h" + +#include "../util/io.h" + +#ifdef __cplusplus +extern "C" { +#endif + +UCX_TEST(test_io_http_stream_parse_chunk_header_hdronly_first); +UCX_TEST(test_io_http_stream_parse_chunk_header_hdronly); +UCX_TEST(test_io_http_stream_parse_chunk_header_hdronly_seq_fail); +UCX_TEST(test_io_http_stream_parse_chunk_header_hdr_data); +UCX_TEST(test_io_http_stream_parse_chunk_header_empty); +UCX_TEST(test_io_http_stream_parse_chunk_header_partial_first); +UCX_TEST(test_io_http_stream_parse_chunk_header_partial); +UCX_TEST(test_io_http_stream_parse_chunk_header_invalid); +UCX_TEST(test_io_http_stream_parse_chunk_header_zero); + +UCX_TEST(test_io_httpstream_write); +UCX_TEST(test_io_httpstream_chunked_write); +UCX_TEST(test_io_httpstream_chunked_write_xx); +UCX_TEST(test_io_httpstream_chunked_write_end); +UCX_TEST(test_io_httpstream_chunked_write_partial_header); +UCX_TEST(test_io_httpstream_chunked_write_partial_data); +UCX_TEST(test_io_httpstream_chunked_write_partial_trailer); +UCX_TEST(test_io_httpstream_chunked_write_partial_trailer_partial_header); +UCX_TEST(test_io_httpstream_chunked_write_data_2x); +UCX_TEST(test_io_httpstream_chunked_write_xx_limit); + + +#ifdef __cplusplus +} +#endif + +#endif /* TEST_IO_H */ +
--- a/src/server/test/main.c Wed May 31 19:39:10 2023 +0200 +++ b/src/server/test/main.c Sun Jun 04 20:09:18 2023 +0200 @@ -48,6 +48,7 @@ #include "webdav.h" #include "uri.h" #include "object.h" +#include "io.h" void register_pg_tests(int argc, char **argv, UcxTestSuite *suite); @@ -99,6 +100,27 @@ ucx_test_register(suite, test_expr_parse_expr_func_expr1); ucx_test_register(suite, test_expr_parse_expr_func_expr2); + // io tests + ucx_test_register(suite, test_io_http_stream_parse_chunk_header_hdronly_first); + ucx_test_register(suite, test_io_http_stream_parse_chunk_header_hdronly); + ucx_test_register(suite, test_io_http_stream_parse_chunk_header_hdronly_seq_fail); + ucx_test_register(suite, test_io_http_stream_parse_chunk_header_hdr_data); + ucx_test_register(suite, test_io_http_stream_parse_chunk_header_empty); + ucx_test_register(suite, test_io_http_stream_parse_chunk_header_partial_first); + ucx_test_register(suite, test_io_http_stream_parse_chunk_header_partial); + ucx_test_register(suite, test_io_http_stream_parse_chunk_header_invalid); + ucx_test_register(suite, test_io_http_stream_parse_chunk_header_zero); + ucx_test_register(suite, test_io_httpstream_write); + ucx_test_register(suite, test_io_httpstream_chunked_write); + ucx_test_register(suite, test_io_httpstream_chunked_write_end); + ucx_test_register(suite, test_io_httpstream_chunked_write_xx); + ucx_test_register(suite, test_io_httpstream_chunked_write_partial_header); + ucx_test_register(suite, test_io_httpstream_chunked_write_partial_data); + ucx_test_register(suite, test_io_httpstream_chunked_write_partial_trailer); + ucx_test_register(suite, test_io_httpstream_chunked_write_partial_trailer_partial_header); + ucx_test_register(suite, test_io_httpstream_chunked_write_data_2x); + ucx_test_register(suite, test_io_httpstream_chunked_write_xx_limit); + // vfs tests ucx_test_register(suite, test_vfs_open); ucx_test_register(suite, test_vfs_mkdir);
--- a/src/server/test/objs.mk Wed May 31 19:39:10 2023 +0200 +++ b/src/server/test/objs.mk Sun Jun 04 20:09:18 2023 +0200 @@ -39,6 +39,7 @@ TESTOBJ += writer.o TESTOBJ += uri.o TESTOBJ += object.o +TESTOBJ += io.o TESTOBJS = $(TESTOBJ:%=$(TEST_OBJPRE)%) TESTSOURCE = $(TESTOBJ:%.o=test/%.c)
--- a/src/server/test/testutils.c Wed May 31 19:39:10 2023 +0200 +++ b/src/server/test/testutils.c Sun Jun 04 20:09:18 2023 +0200 @@ -28,6 +28,7 @@ #include <stdio.h> #include <stdlib.h> +#include <limits.h> #include <cx/string.h> #include <cx/utils.h> @@ -121,13 +122,32 @@ } -static ssize_t test_io_write(IOStream *io, void *buf, size_t size) { +static ssize_t test_io_write(IOStream *io, const void *buf, size_t size) { TestIOStream *st = (TestIOStream*)io; + if(size > st->max_write) size = st->max_write; return cxBufferWrite(buf, 1, size, st->buf); } static ssize_t test_io_writev(IOStream *io, struct iovec *iovec, int iovctn) { - return -1; + TestIOStream *st = (TestIOStream*)io; + ssize_t wv = 0; + for(int i=0;i<iovctn;i++) { + ssize_t available = st->max_write - wv; + size_t len = iovec[i].iov_len; + if(len > available) { + len = available; + } + + ssize_t w = test_io_write(io, iovec[i].iov_base, len); + if(w <= 0) { + break; + } + wv += w; + if(wv >= st->max_write) { + break; + } + } + return wv; } static ssize_t test_io_read(IOStream *io, void *buf, size_t size) { @@ -157,6 +177,7 @@ flags = CX_BUFFER_AUTO_EXTEND|CX_BUFFER_FREE_CONTENTS; } stream->buf = malloc(sizeof(CxBuffer)); + stream->max_write = INT_MAX; cxBufferInit(stream->buf, NULL, size, cxDefaultAllocator, flags); stream->io.st.write = test_io_write;
--- a/src/server/test/testutils.h Wed May 31 19:39:10 2023 +0200 +++ b/src/server/test/testutils.h Sun Jun 04 20:09:18 2023 +0200 @@ -43,7 +43,8 @@ typedef struct TestIOStream { HttpStream io; - CxBuffer *buf; + CxBuffer *buf; + int max_write; } TestIOStream; Session* testutil_session(void);
--- a/src/server/util/io.c Wed May 31 19:39:10 2023 +0200 +++ b/src/server/util/io.c Sun Jun 04 20:09:18 2023 +0200 @@ -118,7 +118,7 @@ } #ifdef XP_UNIX -ssize_t net_sys_write(Sysstream *st, void *buf, size_t nbytes) { +ssize_t net_sys_write(Sysstream *st, const void *buf, size_t nbytes) { return write(st->fd, buf, nbytes); } @@ -264,6 +264,10 @@ st->buflen = NULL; st->bufpos = NULL; st->chunk_buf_pos = 0; + st->current_chunk_length = 0; + st->current_chunk_pos = 0; + st->write_chunk_buf_len = 0; + st->write_chunk_buf_pos = 0; st->chunked_enc = WS_FALSE; st->read_eof = WS_TRUE; st->write_eof = WS_FALSE; @@ -318,30 +322,141 @@ return http->written; } -ssize_t net_http_write(HttpStream *st, void *buf, size_t nbytes) { +/* + * iovec callback func + * returns number of payload bytes written (number of bytes returned back to the net_write caller) + */ +typedef ssize_t(*writeop_finish_func)(HttpStream *st, char *base, size_t len, size_t written, void *udata); + +static ssize_t httpstream_finish_prev_header(HttpStream *st, char *base, size_t len, size_t written, void *udata) { + st->write_chunk_buf_pos += written; + if(st->write_chunk_buf_pos == st->write_chunk_buf_len) { + st->write_chunk_buf_len = 0; + st->write_chunk_buf_pos = 0; + } + return 0; +} + +static ssize_t httpstream_finish_data(HttpStream *st, char *base, size_t len, size_t written, void *udata) { + st->current_chunk_pos += written; + if(st->current_chunk_pos == st->current_chunk_length) { + st->current_chunk_length = 0; + st->current_chunk_pos = 0; + st->current_trailer = 2; + } + return written; +} + +static ssize_t httpstream_finish_new_header(HttpStream *st, char *base, size_t len, size_t written, void *udata) { + size_t *chunk_len = udata; + st->current_chunk_length = *chunk_len; + st->current_chunk_pos = 0; // new chunk started + if(written < len) { + st->write_chunk_buf_len = len-written; + st->write_chunk_buf_pos = 0; + memcpy(st->write_chunk_buf + st->write_chunk_buf_pos, base+written, st->write_chunk_buf_len); + } else { + st->write_chunk_buf_len = 0; + st->write_chunk_buf_pos = 0; + } + return 0; +} + +static ssize_t httpstream_finish_trailer(HttpStream *st, char *base, size_t len, size_t written, void *udata) { + st->current_trailer -= written; + return 0; +} + +ssize_t net_http_write(HttpStream *st, const void *buf, size_t nbytes) { if(st->write_eof) return 0; IOStream *fd = st->fd; - if(st->chunked_enc) { - // TODO: on some plattforms iov_len is smaller than size_t - struct iovec io[3]; - char chunk_len[16]; - io[0].iov_base = chunk_len; - io[0].iov_len = snprintf(chunk_len, 16, "%zx\r\n", nbytes); - io[1].iov_base = buf; - io[1].iov_len = nbytes; - io[2].iov_base = "\r\n"; - io[2].iov_len = 2; - // TODO: FIXME: if wv < sum of iov_len, everything would explode - // we need to store the chunk state and remaining bytes - // TODO: FIX IT NOW, IT IS HORRIBLE BROKEN - ssize_t wv = fd->writev(fd, io, 3); - ssize_t w = wv - io[0].iov_len - io[2].iov_len; + if(!st->chunked_enc) { + ssize_t w = fd->write(fd, buf, nbytes); st->written += w > 0 ? w : 0; return w; } else { - ssize_t w = fd->write(fd, buf, nbytes); - st->written += w > 0 ? w : 0; - return w; + struct iovec io[8]; + writeop_finish_func io_finished[8]; + void *io_finished_udata[8]; + int iovec_len = 0; + + char *str_crlf = "\r\n"; + + size_t prev_chunk_len = st->current_chunk_length; + size_t new_chunk_len = 0; + + // was the previous chunk header completely sent? + if(st->write_chunk_buf_len > 0) { + io[0].iov_base = &st->write_chunk_buf[st->write_chunk_buf_pos]; + io[0].iov_len = st->write_chunk_buf_len - st->write_chunk_buf_pos; + io_finished[0] = httpstream_finish_prev_header; + io_finished_udata[0] = &prev_chunk_len; + iovec_len++; + } + + // was the previous chunk payload completely sent? + if(st->current_chunk_length != 0) { + size_t chunk_remaining = st->current_chunk_length - st->current_chunk_pos; + size_t prev_nbytes = chunk_remaining > nbytes ? nbytes : chunk_remaining; + io[iovec_len].iov_base = (char*)buf; + io[iovec_len].iov_len = prev_nbytes; + io_finished[iovec_len] = httpstream_finish_data; + buf = ((char*)buf) + prev_nbytes; + nbytes -= prev_nbytes; + iovec_len++; + + io[iovec_len].iov_base = str_crlf; + io[iovec_len].iov_len = 2; + io_finished[iovec_len] = httpstream_finish_trailer; + iovec_len++; + } else if(st->current_trailer > 0) { + io[iovec_len].iov_base = str_crlf + 2 - st->current_trailer; + io[iovec_len].iov_len = st->current_trailer; + io_finished[iovec_len] = httpstream_finish_trailer; + iovec_len++; + } + + // TODO: on some plattforms iov_len is smaller than size_t + // if nbytes > INT_MAX, it should be devided into multiple + // iovec entries + char chunk_len[16]; + if(nbytes > 0) { + new_chunk_len = nbytes; + io[iovec_len].iov_base = chunk_len; + io[iovec_len].iov_len = snprintf(chunk_len, 16, "%zx\r\n", nbytes); + io_finished[iovec_len] = httpstream_finish_new_header; + io_finished_udata[iovec_len] = &new_chunk_len; + iovec_len++; + + io[iovec_len].iov_base = (char*)buf; + io[iovec_len].iov_len = nbytes; + io_finished[iovec_len] = httpstream_finish_data; + iovec_len++; + + io[iovec_len].iov_base = str_crlf; + io[iovec_len].iov_len = 2; + io_finished[iovec_len] = httpstream_finish_trailer; + iovec_len++; + } + + ssize_t wv = fd->writev(fd, io, iovec_len); + if(wv <= 0) { + return wv; + } + + size_t ret_w = 0; + int i = 0; + while(wv > 0) { + char *base = io[i].iov_base; + size_t len = io[i].iov_len; + size_t wlen = wv > len ? len : wv; + ret_w += io_finished[i](st, base, len, wlen, io_finished_udata[i]); + wv -= wlen; + i++; + } + + st->written += ret_w; + return ret_w; } } @@ -453,7 +568,7 @@ * -1 if an error occured * >0 chunk header length */ -static int parse_chunk_header(char *str, int len, WSBool first, int64_t *chunklen) { +int http_stream_parse_chunk_header(char *str, int len, WSBool first, int64_t *chunklen) { char *hdr_start = NULL; char *hdr_end = NULL; int i = 0; @@ -569,7 +684,7 @@ } int chunkbuf_len = st->chunk_buf_pos + r; int64_t chunklen; - int ret = parse_chunk_header(st->chunk_buf, chunkbuf_len, st->read_total > 0 ? FALSE : TRUE, &chunklen); + int ret = http_stream_parse_chunk_header(st->chunk_buf, chunkbuf_len, st->read_total > 0 ? FALSE : TRUE, &chunklen); if(ret == 0) { // incomplete chunk header st->chunk_buf_pos = chunkbuf_len; @@ -651,7 +766,7 @@ return (IOStream*)st; } -ssize_t net_ssl_write(SSLStream *st, void *buf, size_t nbytes) { +ssize_t net_ssl_write(SSLStream *st, const void *buf, size_t nbytes) { int ret = SSL_write(st->ssl, buf, nbytes); if(ret <= 0) { st->error = SSL_get_error(st->ssl, ret); @@ -734,7 +849,7 @@ return r; } -ssize_t net_write(SYS_NETFD fd, void *buf, size_t nbytes) { +ssize_t net_write(SYS_NETFD fd, const void *buf, size_t nbytes) { ssize_t r = ((IOStream*)fd)->write(fd, buf, nbytes); if(r < 0) { ((IOStream*)fd)->io_errno = errno;
--- a/src/server/util/io.h Wed May 31 19:39:10 2023 +0200 +++ b/src/server/util/io.h Sun Jun 04 20:09:18 2023 +0200 @@ -59,7 +59,7 @@ typedef struct Sysstream Sysstream; typedef struct HttpStream HttpStream; -typedef ssize_t(*io_write_f)(IOStream *, void *, size_t); +typedef ssize_t(*io_write_f)(IOStream *, const void *, size_t); typedef ssize_t(*io_writev_f)(IOStream *, struct iovec *, int); typedef ssize_t(*io_read_f)(IOStream *, void *, size_t); typedef ssize_t(*io_sendfile_f)(IOStream *, sendfiledata *); @@ -136,6 +136,35 @@ */ WSBool chunked_enc; /* + * current chunk size (set after the header is sent) + */ + size_t current_chunk_length; + /* + * current chunk position + */ + size_t current_chunk_pos; + /* + * missing trailer before new data + * 0: no trailer + * 2: crlf + * 1: lf + */ + int current_trailer; + /* + * write chunk header buffer + */ + char write_chunk_buf[HTTP_STREAM_CBUF_SIZE]; + /* + * chunk header buffer length + * only used when the chunk header was completely sent + * must be 0 before payload data is sent + */ + int write_chunk_buf_len; + /* + * current write_chunk_buf position (if remaining != 0) + */ + int write_chunk_buf_pos; + /* * end of file indicator (read) */ WSBool read_eof; @@ -156,7 +185,7 @@ /* system stream */ IOStream* Sysstream_new(pool_handle_t *pool, SYS_SOCKET fd); -ssize_t net_sys_write(Sysstream *st, void *buf, size_t nbytes); +ssize_t net_sys_write(Sysstream *st, const void *buf, size_t nbytes); ssize_t net_sys_writev(Sysstream *st, struct iovec *iovec, int iovcnt); ssize_t net_sys_read(Sysstream *st, void *buf, size_t nbytes); ssize_t net_sys_sendfile(Sysstream *st, sendfiledata *sfd); @@ -173,7 +202,7 @@ WSBool httpstream_eof(IOStream *st); int64_t httpstream_written(IOStream *st); -ssize_t net_http_write(HttpStream *st, void *buf, size_t nbytes); +ssize_t net_http_write(HttpStream *st, const void *buf, size_t nbytes); ssize_t net_http_writev(HttpStream *st, struct iovec *iovec, int iovcnt); ssize_t net_http_read(HttpStream *st, void *buf, size_t nbytes); ssize_t net_http_read_chunked(HttpStream *st, void *buf, size_t nbytes); @@ -183,10 +212,12 @@ void net_http_setmode(HttpStream *st, int mode); int net_http_poll(HttpStream *st, EventHandler *ev, int events, Event *cb); +int http_stream_parse_chunk_header(char *str, int len, WSBool first, int64_t *chunklen); + /* ssl stream */ IOStream* sslstream_new(pool_handle_t *pool, SSL *ssl); -ssize_t net_ssl_write(SSLStream *st, void *buf, size_t nbytes); +ssize_t net_ssl_write(SSLStream *st, const void *buf, size_t nbytes); ssize_t net_ssl_writev(SSLStream *st, struct iovec *iovec, int iovcnt); ssize_t net_ssl_read(SSLStream *st, void *buf, size_t nbytes); void net_ssl_close(SSLStream *st);
