# HG changeset patch # User Olaf Wintermann # Date 1651869953 -7200 # Node ID bb536d4bc174f1ba0ee3a37612eddb4a3fca83ce # Parent 6f82ede01e1cb656b6c9106c6fc12c6f987b05d8 first semi-functional implementation of chunked transfer encoding for request bodies diff -r 6f82ede01e1c -r bb536d4bc174 src/server/daemon/httprequest.c --- a/src/server/daemon/httprequest.c Thu May 05 21:46:10 2022 +0200 +++ b/src/server/daemon/httprequest.c Fri May 06 22:45:53 2022 +0200 @@ -396,6 +396,21 @@ net_io->max_read = ctlen - cur_input_available; } } + char *transfer_encoding = pblock_findkeyval(pb_key_transfer_encoding, rq->rq.headers); + if(transfer_encoding) { + if(!strcmp(transfer_encoding, "chunked")) { + netbuf *nb = sn->netbuf; + sn->buffer = (char*)nb->inbuf; + sn->pos = nb->pos; + sn->cursize = nb->cursize; + + if(httpstream_enable_chunked_read(sn->sn.csd, sn->buffer, nb->maxsize, &sn->cursize, &sn->pos)) { + pool_destroy(pool); + // TODO: error 500 + return 1; + } + } + } // // Send the request to the NSAPI system diff -r 6f82ede01e1c -r bb536d4bc174 src/server/daemon/session.h --- a/src/server/daemon/session.h Thu May 05 21:46:10 2022 +0200 +++ b/src/server/daemon/session.h Fri May 06 22:45:53 2022 +0200 @@ -47,6 +47,10 @@ threadpool_t *currentpool; threadpool_t *defaultpool; + char *buffer; + int pos; + int cursize; + UcxAllocator allocator; ServerConfiguration *config; diff -r 6f82ede01e1c -r bb536d4bc174 src/server/util/io.c --- a/src/server/util/io.c Thu May 05 21:46:10 2022 +0200 +++ b/src/server/util/io.c Fri May 06 22:45:53 2022 +0200 @@ -253,10 +253,49 @@ st->max_read = 0; st->read = 0; st->chunked_enc = WS_FALSE; - st->buffered = WS_FALSE; + st->chunk_buf_pos = 0; return (IOStream*)st; } +int httpstream_enable_chunked_read(IOStream *st, char *buffer, size_t bufsize, int *cursize, int *pos) { + if(st->read != (io_read_f)net_http_read) { + log_ereport(LOG_FAILURE, "%s", "httpstream_enable_chunked_read: IOStream is not an HttpStream"); + return 1; + } + st->read = (io_read_f)net_http_read_chunked; + HttpStream *http = (HttpStream*)st; + http->max_read = 0; + http->read = 0; + http->readbuf = buffer; + http->bufsize = bufsize; + http->buflen = *cursize; // TODO: store buflen as pointer + http->bufpos = pos; + http->chunk_buf_pos = 0; + http->remaining_len = 0; + http->remaining_pos = 0; + return 0; +} + +int httpstream_enable_chunked_write(IOStream *st) { + if(st->write != (io_write_f)net_http_write) { + log_ereport(LOG_FAILURE, "%s", "httpstream_enable_chunked_write: IOStream is not an HttpStream"); + return 1; + } + HttpStream *http = (HttpStream*)st; + http->chunked_enc = WS_TRUE; + return 0; +} + +int httpstream_set_max_read(IOStream *st, int64_t maxread) { + if(st->write != (io_write_f)net_http_write) { + log_ereport(LOG_FAILURE, "%s", "httpstream_set_max_read: IOStream is not an HttpStream"); + return 1; + } + HttpStream *http = (HttpStream*)st; + http->max_read = maxread; + return 0; +} + ssize_t net_http_write(HttpStream *st, void *buf, size_t nbytes) { IOStream *fd = st->fd; if(st->chunked_enc) { @@ -308,10 +347,246 @@ return 0; } ssize_t r = st->fd->read(st->fd, buf, nbytes); + if(r < 0) { + st->st.io_errno = st->fd->io_errno; + } st->read += r; return r; } +#define BUF_UNNEEDED_DIFF 64 +/* + * read from st->chunk_buf first, read from st->fd if perform_io is true + */ +static ssize_t net_http_read_buffered(HttpStream *st, char *buf, size_t nbytes, WSBool read_data, WSBool *perform_io) { + ssize_t r = 0; + + // remaining bytes from the chunkbuf + if(st->remaining_len > 0) { + size_t cplen = st->remaining_len > nbytes ? nbytes : st->remaining_len; + WSBool ret = FALSE; + if(read_data) { + // if we read data (and not a chunk header), we limit the + // amount of bytes we copy + size_t chunk_available = st->max_read - st->read; + if(cplen > chunk_available) { + cplen = chunk_available; + ret = TRUE; + } + st->read += cplen; + } + memcpy(buf, &st->remaining_buf[st->remaining_pos], cplen); + st->remaining_pos += cplen; + st->remaining_len -= cplen; + buf += cplen; + nbytes -= cplen; + r += cplen; + if(st->remaining_len == 0) { + st->remaining_pos = 0; + } + if(ret) { + return r; + } + } + + // copy available data from st->readbuf to buf + int pos = *st->bufpos; + size_t buf_available = st->buflen - pos; + if(buf_available) { + size_t cplen = buf_available > nbytes ? nbytes : buf_available; + if(read_data) { + // if we read data (and not a chunk header), we limit the + // amount of bytes we copy + size_t chunk_available = st->max_read - st->read; + cplen = cplen > chunk_available ? chunk_available : cplen; + st->read += cplen; + } + memcpy(buf, st->readbuf + pos, cplen); + *st->bufpos += cplen; + r += cplen; + buf += cplen; + nbytes -= cplen; + } + + if(*perform_io && nbytes > 0) { + // fill buffer again + ssize_t rlen = st->fd->read(st->fd, st->readbuf, st->bufsize); + st->buflen = rlen; + *st->bufpos = 0; + *perform_io = WS_FALSE; + if(rlen < 0) { + st->st.io_errno = st->fd->io_errno; + } + + if(rlen > 0) { + // call func again to get data from buffer (no IO will be performed) + r += net_http_read_buffered(st, buf, nbytes, read_data, perform_io); + } + } + + return r; +} + + +/* + * parses a chunk header + * the chunk length is stored in chunklen + * return: 0 if the data is incomplete + * -1 if an error occured + * >0 chunk header length + */ +static int parse_chunk_header(char *str, int len, WSBool first, int64_t *chunklen) { + char *hdr_start = NULL; + char *hdr_end = NULL; + int i = 0; + if(first) { + hdr_start = str; + } else { + if(len < 3) { + return 0; + } + if(str[0] == '\r' && str[1] == '\n') { + hdr_start = str+2; + i = 2; + } else if(str[0] == '\n') { + hdr_start = str+1; + i = 1; + } else { + return -1; + } + } + + for(;i= len) { + return 0; + } + if(str[i] == '\n') { + i++; + } else if(str[i] == '\r') { + if(++i >= len) { + return 0; + } + if(str[i] == '\n') { + i++; + } else { + return -1; + } + } else { + return -1; + } + } + + *chunklen = clen; + return i; +} + +ssize_t net_http_read_chunked(HttpStream *st, void *buf, size_t nbytes) { + if(st->read_eof) { + return 0; + } + + char *rbuf = buf; // buffer pos + size_t rd = 0; // number of bytes read + size_t rbuflen = nbytes; // number of bytes until end of buf + WSBool perform_io = WS_TRUE; // we do only 1 read before we abort + while(rd < nbytes && (perform_io || (st->max_read - st->read) > 0)) { + // how many bytes are available in the current chunk + size_t chunk_available = st->max_read - st->read; + if(chunk_available > 0) { + ssize_t r = net_http_read_buffered(st, rbuf, rbuflen, TRUE, &perform_io); + if(r == 0) { + break; + } + rd += r; + st->read_total += r; + rbuf += r; + rbuflen -= r; + } else { + int chunkbuf_avail = HTTP_STREAM_CBUF_SIZE - st->chunk_buf_pos; + if(chunkbuf_avail == 0) { + // for some reason HTTP_STREAM_CBUF_SIZE is not enough + // to store the chunk header + // this indicates that something has gone wrong (or this is an attack) + st->read_eof = WS_TRUE; + return -1; + } + // fill st->chunk_buf + ssize_t r = net_http_read_buffered(st, &st->chunk_buf[st->chunk_buf_pos], chunkbuf_avail, FALSE, &perform_io); + if(r == 0) { + break; + } + int chunkbuf_len = st->chunk_buf_pos + r; + int64_t chunklen; + int ret = parse_chunk_header(st->chunk_buf, chunkbuf_len, st->read_total > 0 ? FALSE : TRUE, &chunklen); + if(ret == 0) { + // incomplete chunk header + st->chunk_buf_pos = chunkbuf_len; + } else if(ret < 0) { + // error + st->read_eof = WS_TRUE; + return -1; + } else if(ret > 0) { + st->max_read = chunklen; + st->read = 0; + st->remaining_len = chunkbuf_len - ret; + if(st->remaining_len > 0) { + memcpy(st->remaining_buf, st->chunk_buf, HTTP_STREAM_CBUF_SIZE); + st->remaining_pos = st->chunk_buf_pos + ret; + } else { + st->remaining_pos = 0; + } + st->remaining_len = chunkbuf_len - ret; + st->chunk_buf_pos = 0; + + if(chunklen == 0) { + st->read_eof = WS_TRUE; + break; + } + } + } + + if(!perform_io && rd == 0) { + perform_io = WS_TRUE; + } + } + + return rd; +} + ssize_t net_http_sendfile(HttpStream *st, sendfiledata *sfd) { ssize_t ret = 0; // TODO: support chunked transfer encoding diff -r 6f82ede01e1c -r bb536d4bc174 src/server/util/io.h --- a/src/server/util/io.h Thu May 05 21:46:10 2022 +0200 +++ b/src/server/util/io.h Fri May 06 22:45:53 2022 +0200 @@ -89,13 +89,24 @@ #endif }; +#define HTTP_STREAM_CBUF_SIZE 16 struct HttpStream { IOStream st; IOStream *fd; uint64_t max_read; uint64_t read; + uint64_t read_total; + char *readbuf; + size_t bufsize; // allocated buffer size + size_t buflen; // currently number of bytes in the buffer + int *bufpos; // current buffer position + int chunk_buf_pos; + char chunk_buf[HTTP_STREAM_CBUF_SIZE]; + char remaining_buf[HTTP_STREAM_CBUF_SIZE]; + int remaining_len; + int remaining_pos; WSBool chunked_enc; - WSBool buffered; + WSBool read_eof; }; typedef struct SSLStream { @@ -120,9 +131,14 @@ /* http stream */ IOStream* httpstream_new(pool_handle_t *pool, IOStream *fd); +int httpstream_enable_chunked_read(IOStream *st, char *buffer, size_t bufsize, int *cursize, int *pos); +int httpstream_enable_chunked_write(IOStream *st); +int httpstream_set_max_read(IOStream *st, int64_t maxread); + ssize_t net_http_write(HttpStream *st, void *buf, size_t nbytes); ssize_t net_http_writev(HttpStream *st, struct iovec *iovec, int iovcnt); ssize_t net_http_read(HttpStream *st, void *buf, size_t nbytes); +ssize_t net_http_read_chunked(HttpStream *st, void *buf, size_t nbytes); ssize_t net_http_sendfile(HttpStream *st, sendfiledata *sfd); void net_http_close(HttpStream *st); void net_http_finish(HttpStream *st);