Attachment #88540 for bug #125350

View | Details | Raw Unified | Return to bug 125350
Collapse All | Expand All

Lines 4-10 Link Here

(-)src/lib/libfetch/Makefile (-1 / +3 lines)
4		4
5	LIB= fetch	5	LIB= fetch
6	CFLAGS+= -I.	6	CFLAGS+= -I.
7	SRCS= fetch.c common.c ftp.c http.c file.c \	7	SRCS= fetch.c common.c ftp.c http.c httpdecode.c file.c \
8	ftperr.h httperr.h	8	ftperr.h httperr.h
9	INCS= fetch.h	9	INCS= fetch.h
10	MAN= fetch.3	10	MAN= fetch.3
Lines 20-25 Link Here
20	LDADD= -lssl -lcrypto	20	LDADD= -lssl -lcrypto
21	.endif	21	.endif
22		22
		23	LDADD+= -lz
		24
23	CFLAGS+= -DFTP_COMBINE_CWDS	25	CFLAGS+= -DFTP_COMBINE_CWDS
24		26
25	CSTD?= c99	27	CSTD?= c99




#include "fetch.h"
#include "common.h"
#include "httperr.h"
#include "httpdecode.h"

/* Maximum number of redirects to follow */
#define MAX_REDIRECT 5

	hdr_error = -1,
	hdr_end = 0,
	hdr_unknown = 1,
	hdr_content_encoding,
	hdr_content_length,
	hdr_content_range,
	hdr_last_modified,

	hdr_t		 num;
	const char	*name;
} hdr_names[] = {
	{ hdr_content_encoding,		"Content-Encoding" },
	{ hdr_content_length,		"Content-Length" },
	{ hdr_content_range,		"Content-Range" },
	{ hdr_last_modified,		"Last-Modified" },

}

/*
 * Parse a content-encoding header
 */
static int
http_parse_encoding(const char *p)
{
	if (strcmp("gzip", p) == 0)
		return(ENCODING_GZIP);
	if (strcmp("deflate", p) == 0)
		return(ENCODING_DEFLATE);
	if (strcmp("compress", p) == 0)
		return(ENCODING_COMPRESS);
	return(ENCODING_RAW);
}

/*
 * Parse a content-length header
 */
static int

	conn_t *conn;
	struct url *url, *new;
	int chunked, direct, need_auth, noredirect, verbose;
	int e, i, n, val, encoding;
	off_t offset, clength, length, size;
	time_t mtime;
	const char *p;
	FILE *f, *d;
	hdr_t h;
	char hbuf[MAXHOSTNAMELEN + 7], *host;

	f = NULL;
	d = NULL;

	direct = CHECK_FLAG('d');
	noredirect = CHECK_FLAG('A');
	verbose = CHECK_FLAG('v');

		length = -1;
		size = -1;
		mtime = 0;
		encoding = ENCODING_RAW;

		/* check port */
		if (!url->port)

			http_cmd(conn, "User-Agent: %s " _LIBFETCH_VER, getprogname());
		if (url->offset > 0)
			http_cmd(conn, "Range: bytes=%lld-", (long long)url->offset);
		http_cmd(conn, "Accept-Encoding: gzip,deflate");
		http_cmd(conn, "Connection: close");
		http_cmd(conn, "");


			case hdr_error:
				http_seterr(HTTP_PROTOCOL_ERROR);
				goto ouch;
			case hdr_content_encoding:
				encoding = http_parse_encoding(p);
				break;
			case hdr_content_length:
				http_parse_length(p, &clength);
				break;


	/* fill in stats */
	if (us) {
		/* we can only predict the size of unencoded streams */
		if (encoding == ENCODING_RAW)
			us->size = size;
		us->atime = us->mtime = mtime;
	}


		goto ouch;
	}

	/* wrap the decoder around it */
	if ((d = httpDecode(f, encoding, size)) == NULL) {
		fetch_syserr();
		fclose(f);
		goto ouch;
	}

	if (url != URL)
		fetchFreeURL(url);
	if (purl)

		f = NULL;
	}

	return (d);

ouch:
	if (url != URL)




/*
 * I wrote this and I say you can do whatever you want with it. Period.
 * However, I'd love to hear from you what you've done.
 *
 * Dominic Fandrey <kamikaze@bsdforen.de>
 */

/**
 * \file httpdecode.c
 *
 * This file contains the implemention of the prototypes defined in
 * httpdecode.h.
 *
 * @brief
 *	HTTP content decoding implemention.
 * @see
 *	httpdecode.h
 * @author
 *	Dominic Fandrey <kamikaze@bsdforen.de>
 * @version
 *	0.1.99.2008.07.07
 */

/* LINTLIBRARY */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
#include <zlib.h>
#include "httpdecode.h"

/* PRIVATE STRUCTS */

/**
 * @brief
 *	The necessary data to maintain a zlib decoding stream.
 */
struct zlibStream {
	/**
	 * @brief
	 *	The original stream with the encoded data.
	 */
	FILE * source;

	/**
	 * @brief
	 *	A read buffer for the encoded stream.
	 */
	char * buffer;

	/**
	 * This specifies the encoding of the data. The values
	 *  ENCODING_GZIP and ENCODING_DEFLATE are possible.
	 *
	 * @brief
	 *	The encoding type of the stream.
	 */
	int encoding;

	/**
	 * @brief
	 *	The stream data used by zlib.
	 */
	z_stream stream;

	/**
	 * The length of the source stream. The value 0 means that the length
	 * is unknown higher values will be used to automatically close the
	 * stream. This prevents overreading and allows the continued use
	 * of the underlying HTTP stream.
	 *
	 * @brief
	 *	The length of the encoded source stream.
	 */
	size_t length;

	/**
	 * @brief
	 *	The amount of data that has been read.
	 */
	size_t read;

	/**
	 * @brief
	 *	The size of the buffer for encoded data.
	 */
	size_t bufferSize;

	/**
	 * @brief
	 *	The amount of data available in the buffer.
	 */
	size_t bufferUsed;
};

/* PRIVATE PROTOTYPES */
void moveBuffer(struct zlibStream * cookie, char * newBuffer, size_t size);
FILE * zlibOpen(struct zlibStream * cookie);
size_t zlibRead(struct zlibStream * cookie, char * buffer, size_t length);
int zlibClose(struct zlibStream * cookie);
/* TODO
FILE * compressOpen(struct zlibStream * cookie);
size_t compressRead(struct zlibStream * cookie, char * buffer, size_t length);
int compressClose(struct zlibStream * cookie);
FILE * randomOpen(struct zlibStream * cookie);
size_t randomRead(struct zlibStream * cookie, char * buffer, size_t length);
int randomSeek(struct zlibStream * cookie, off_t offset, int whence);
int randomClose(struct zlibStream * cookie);
*/

/* PUBLIC FUNCTIONS */

/**
 * Opens a given stream for decoding and returns a FILE handle that can be
 * used with the fread and fclose function. Internally funopen is used
 * to achieve this.
 *
 * In case of failure NULL is returned and errno is set to EINVAL for
 * invalid parameters and ENOMEM for insufficient memory.
 *
 * @brief
 *	Open a FILE stream to read decoded data from.
 * @param source
 *	The stream to read the encoded data from.
 * @param encoding
 *	The encoding type of the source stream.
 * @param length
 *	The length of the source stream. Use 0 if unknown.
 * @return
 *	Returns a FILE handle to read an encoded stream.
 * @see
 *	funopen(3)
 * @see
 *	fread(3)
 * @see
 *	fclose(3)
 */
FILE * httpDecode(FILE * source, int encoding, size_t length) {
	struct zlibStream * zlibCookie;

	switch (encoding) {
	case ENCODING_RAW:
		return(source);
	case ENCODING_GZIP: case ENCODING_DEFLATE:
		zlibCookie = malloc(sizeof(struct zlibStream));
		if (zlibCookie == NULL) /* errno == ENOMEM */
			return(NULL);
		zlibCookie->buffer = NULL;
		zlibCookie->bufferSize = 0;
		zlibCookie->source = source;
		zlibCookie->length = length;
		zlibCookie->read = 0;
		zlibCookie->encoding = encoding;
		return(zlibOpen(zlibCookie));
	case ENCODING_COMPRESS:
		return(NULL);
	}

	return(NULL);
}

/**
 * This function is a wraper around httpDecode that allows random access
 * by writing the stream into a temporary file. The file is buffered
 * by a given number of buffers in memory.
 * Buffers are overwritten in LRU order.
 *
 * @brief
 *	A file backed wrapper around httpDecode for random access.
 * @param source
 *	The stream to read the encoded data from.
 * @param encoding
 *	The encoding type of the source stream.
 * @param length
 *	The length of the source stream. Use 0 if unknown.
 * @param bufferSize
 *	The size of a buffer.
 * @param
 *	The number of buffers.
 * @return
 *	Returns a FILE handle to read an encoded stream.
 */
/* TODO
FILE * httpDecodeRandom(FILE * source, int encoding, size_t length,
	size_t bufferSize, size_t buffers) {
	return(source);
}
*/

/* PRIVATE FUNCTIONS */

/**
 * This function replaces the read buffer in the cookie with the new buffer.
 * The old buffer is freed but the contents are saved in the new buffer.
 * However, no security checks are performed.
 * That means that newBuffer must at least have the same size as the old one.
 *
 * @brief
 *	Replace the current read buffer.
 * @param cookie
 *	Contains all the data necessary to maintain the stream.
 * @param newBuffer
 *	The new buffer to use.
 */
void moveBuffer(struct zlibStream * cookie, char * newBuffer, size_t size) {
	memmove(newBuffer, cookie->buffer, cookie->bufferUsed);
	free(cookie->buffer);
	cookie->buffer = newBuffer;
	cookie->bufferSize = size;
}

/**
 * This function initializes a zlib stream and creates the file handler
 * that will later be used to pull data from the stream.
 *
 * Upon any kind of failure errno is set to one of the following values:
 * EINVAL	This can either indicate that an unsupported encoding
 *		was given or that this code and the used zlib implemention
 *		are incompatible.
 * ENOMEM	Indicates that the available memory is insuficient for
 *		the decode buffer, zlib or funopen.
 *
 * @brief
 *	Open a zlib stream.
 * @param cookie
 *	Contains all the data necessary to maintain the stream.
 * @return
 *	A FILE* pointer or NULL in case of failure.
 */
FILE * zlibOpen(struct zlibStream * cookie) {
	int wbits;
	z_stream * stream = &(cookie->stream);

	/* Initialize zlib stream data. */
	stream->zalloc = Z_NULL;
	stream->zfree = Z_NULL;
	stream->opaque = Z_NULL;
	stream->avail_in = 0;
	stream->next_in = (Bytef *) cookie->buffer;

	/* Set window bits for the selected encoding. */
	switch(cookie->encoding) {
	case ENCODING_DEFLATE:
		wbits = -MAX_WBITS;
		break;
	case ENCODING_GZIP:
		wbits = MAX_WBITS + 16;
		break;
	default:
		errno = EINVAL;
		return(NULL);
	}

	/* Create the decoding buffer. */
	cookie->bufferSize = 32768; /* Zlib breaks with a smaller buffer. */
	cookie->bufferUsed = 0;
	cookie->buffer = malloc(cookie->bufferSize);
	if (cookie->buffer == NULL)
		return(NULL); /* errno == ENOMEM */

	/* Initialize stream for decoding. */
	switch(inflateInit2(stream, wbits)) {
	case Z_OK:
		errno = 0;
		break;
	case Z_MEM_ERROR:
		errno = ENOMEM;
		break;
	case Z_STREAM_ERROR: /* This is not supposed to happen. */
		errno = EINVAL;
		break;
	}
	if (errno) {
		free(cookie->buffer);
		return(NULL);
	}

	/* Create the file stream to return. */
	return(funopen(cookie,(int (*)(void *, char *, int)) zlibRead,
		NULL, NULL, (int (*)(void *)) zlibClose));
}

/**
 * Writes a chunk of decoded data to the given buffer.
 *
 * In case of an error (size_t) -1 is returned to indicate to the funopen
 * wrapper that an error occured. In such a case errno is set to EIO.
 *
 * An error does not cause the stream to be closed.
 *
 * @brief
 *	Read decoded data from the encoded stream.
 * @param cookie
 *	Contains all the data necessary to maintain the stream.
 * @param buffer
 *	The buffer to write the decoded data to.
 * @param length
 *	The space available in the buffer.
 * @return
 *	The number of bytes written to the buffer or (size_t) -1 in case of
 *	failure.
 */
size_t zlibRead(struct zlibStream * cookie, char * buffer, size_t length) {
	char * tmpBuffer;
	size_t growth, maxRead, bufferAvailable, flushed;
	int zlibStatus;
	z_stream * stream = &(cookie->stream);

	/* 
	 * Adjust buffer size if the target buffer is larger than 2 times
	 * the source buffer.
	 */
	if ((length >> 1) > cookie->bufferSize) {
		tmpBuffer = malloc(length >> 1);

		/*
		 * If creating a new buffer fails pretend never to have
		 * attempted it.
		 */
		if (tmpBuffer == NULL)
			errno = 0;
		else
			/* Move data from the old buffer to the new one. */
			moveBuffer(cookie, tmpBuffer, length >> 1);
	}

	/* Run until the target buffer has been filled. */
	flushed = 0;
	while (length > 0) {
		/* If the input buffer is not full, fill it. */
		growth = 0;
		maxRead = cookie->length - cookie->read;
		bufferAvailable = cookie->bufferSize - cookie->bufferUsed;
		bufferAvailable = (maxRead < bufferAvailable \
			? maxRead : bufferAvailable);
		if (bufferAvailable > 0) {
			growth = fread(cookie->buffer, sizeof(char), \
				bufferAvailable, cookie->source);
			/* Forward errors. */
			if (ferror(cookie->source))
				return((size_t) -1);
			cookie->bufferUsed += growth;
			cookie->read += growth;
		}

		/* Decode data from the read to the target buffer. */
		stream->avail_in = cookie->bufferUsed;
		stream->next_in = (Bytef *) cookie->buffer;
		stream->avail_out = length;
		stream->next_out = (Bytef *) buffer;
		zlibStatus = inflate(stream, Z_SYNC_FLUSH);

		/* The amount of data just written to the target buffer. */
		growth = length - stream->avail_out;

		/* Adjust the read buffer. */
		memmove(cookie->buffer, stream->next_in, \
			(size_t) stream->avail_in);
		cookie->bufferUsed = stream->avail_in;
		stream->next_in = (Bytef *) cookie->buffer;

		/* Adjust the target buffer. */
		flushed += growth;
		buffer += growth;
		length = stream->avail_out;

		/* Deal with errors. */
		switch (zlibStatus) {
		case Z_OK:
			break;
		case Z_STREAM_END:
			length = 0;
			break;
		case Z_BUF_ERROR:
			/* The read buffer is too small, try to double it. */
			tmpBuffer = malloc(cookie->bufferSize << 1);
			if (!tmpBuffer) /* errno == ENOMEN */
				return((size_t) -1);
			moveBuffer(cookie, tmpBuffer, cookie->bufferSize << 1);
			break;
		case Z_NEED_DICT: case Z_DATA_ERROR: case Z_STREAM_ERROR:
			errno = EIO;
			return((size_t) -1);
		case Z_MEM_ERROR:
			errno = ENOMEM;
			return((size_t) -1);
		}
	}

	return(flushed);
}

/**
 * Closes the decoding stream and frees all buffers.
 *
 * @brief
 *	Closes the decoding stream.
 * @param cookie
 *	Contains all the data necessary to maintain the stream.
 * @return
 *	Always 0 for success.
 */
int zlibClose(struct zlibStream * cookie) {
	inflateEnd(&(cookie->stream));
	free(cookie->buffer);
	free(cookie);
	return(0);
}





/*
 * I wrote this and I say you can do whatever you want with it. Period.
 * However, I'd love to hear from you what you've done.
 *
 * Dominic Fandrey <kamikaze@bsdforen.de>
 */

#ifndef HTTPDECODE_H
#define HTTPDECODE_H

/**
 * \file httpdecode.h
 *
 * This file contains the public prototypes and defines required to read
 * compressed data streams. Supported formats are those listed in
 * RFC2616 section 3.5 (HTTP 1.1 content encodings). Compress decoding is
 * not yet implemented.
 *
 * @brief
 *	Public defines and prototypes to decode encoded HTML streams.
 * @see
 *	http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.5
 * @see
 *	httpdecode.c
 * @author
 *	Dominic Fandrey <kamikaze@bsdforen.de>
 * @version
 *	0.3.99.2008.07.07
 */

/**
 * This can be used to abuse httpDecodeRandom as a random (read) access layer
 * for any FILE stream.
 *
 * @brief
 *	The source stream ist not encoded.
 */
#define ENCODING_RAW		0

/**
 * @brief
 *	The source stream is deflate encoded.
 * @see
 *	zlib(3)
 */
#define ENCODING_DEFLATE	1

/**
 * @brief
 *	The source stream is gzip encoded.
 * @see
 *	gzip(1)
 * @see
 *	zlib(3)
 */
#define ENCODING_GZIP		2

/**
 * @brief
 *	The source stream is compress encoded.
 * @see
 *	compress(1)
 */
#define ENCODING_COMPRESS	3


/**
 * Opens a given stream for decoding and returns a FILE handle that can be
 * used with the read and close function. Internally funopen is used
 * to achieve this.
 *
 * @param source
 *	The stream to read the encoded data from.
 * @param encoding
 *	The encoding type of the source stream.
 * @param length
 *	The length of the source stream. Use 0 if unknown.
 * @return
 *	Returns a FILE handle to read an encoded stream.
 * @see
 *	funopen(3)
 * @see
 *	fread(3)
 * @see
 *	fclose(3)
 */
FILE * httpDecode(FILE * source, int encoding, size_t length);

/**
 * This function is a wraper around httpDecode that allows random access
 * by writing the stream into a temporary file. The file is buffered
 * by a given number of buffers in memory.
 * Buffers are overwritten in LRU order.
 *
 * @brief
 *	A file backed wrapper around httpDecode for random access.
 * @param source
 *	The stream to read the encoded data from.
 * @param encoding
 *	The encoding type of the source stream.
 * @param length
 *	The length of the source stream. Use 0 if unknown.
 * @param bufferSize
 *	The size of a buffer.
 * @param
 *	The number of buffers.
 * @return
 *	Returns a FILE handle to read an encoded stream.
 */
/* TODO
FILE * httpDecodeRandom(FILE * source, int encoding, size_t length,
	size_t bufferSize, size_t buffers);
*/

#endif /* HTTPDECODE_H */


Return to bug 125350

Lines 82-87 Link Here

(-)src/lib/libfetch/http.c (-4 / +39 lines)
82	#include "fetch.h"	82	#include "fetch.h"
83	#include "common.h"	83	#include "common.h"
84	#include "httperr.h"	84	#include "httperr.h"
		85	#include "httpdecode.h"
85		86
86	/* Maximum number of redirects to follow */	87	/* Maximum number of redirects to follow */
87	#define MAX_REDIRECT 5	88	#define MAX_REDIRECT 5
Lines 336-341 Link Here
336	hdr_error = -1,	337	hdr_error = -1,
337	hdr_end = 0,	338	hdr_end = 0,
338	hdr_unknown = 1,	339	hdr_unknown = 1,
		340	hdr_content_encoding,
339	hdr_content_length,	341	hdr_content_length,
340	hdr_content_range,	342	hdr_content_range,
341	hdr_last_modified,	343	hdr_last_modified,
Lines 349-354 Link Here
349	hdr_t num;	351	hdr_t num;
350	const char *name;	352	const char *name;
351	} hdr_names[] = {	353	} hdr_names[] = {
		354	{ hdr_content_encoding, "Content-Encoding" },
352	{ hdr_content_length, "Content-Length" },	355	{ hdr_content_length, "Content-Length" },
353	{ hdr_content_range, "Content-Range" },	356	{ hdr_content_range, "Content-Range" },
354	{ hdr_last_modified, "Last-Modified" },	357	{ hdr_last_modified, "Last-Modified" },
Lines 496-501 Link Here
496	}	499	}
497		500
498	/*	501	/*
		502	* Parse a content-encoding header
		503	*/
		504	static int
		505	http_parse_encoding(const char *p)
		506	{
		507	if (strcmp("gzip", p) == 0)
		508	return(ENCODING_GZIP);
		509	if (strcmp("deflate", p) == 0)
		510	return(ENCODING_DEFLATE);
		511	if (strcmp("compress", p) == 0)
		512	return(ENCODING_COMPRESS);
		513	return(ENCODING_RAW);
		514	}
		515
		516	/*
499	* Parse a content-length header	517	* Parse a content-length header
500	*/	518	*/
501	static int	519	static int
Lines 800-813 Link Here
800	conn_t *conn;	818	conn_t *conn;
801	struct url url, new;	819	struct url url, new;
802	int chunked, direct, need_auth, noredirect, verbose;	820	int chunked, direct, need_auth, noredirect, verbose;
803	int e, i, n, val;	821	int e, i, n, val, encoding;
804	off_t offset, clength, length, size;	822	off_t offset, clength, length, size;
805	time_t mtime;	823	time_t mtime;
806	const char *p;	824	const char *p;
807	FILE *f;	825	FILE f, d;
808	hdr_t h;	826	hdr_t h;
809	char hbuf[MAXHOSTNAMELEN + 7], *host;	827	char hbuf[MAXHOSTNAMELEN + 7], *host;
810		828
		829	f = NULL;
		830	d = NULL;
		831
811	direct = CHECK_FLAG('d');	832	direct = CHECK_FLAG('d');
812	noredirect = CHECK_FLAG('A');	833	noredirect = CHECK_FLAG('A');
813	verbose = CHECK_FLAG('v');	834	verbose = CHECK_FLAG('v');
Lines 834-839 Link Here
834	length = -1;	855	length = -1;
835	size = -1;	856	size = -1;
836	mtime = 0;	857	mtime = 0;
		858	encoding = ENCODING_RAW;
837		859
838	/* check port */	860	/* check port */
839	if (!url->port)	861	if (!url->port)
Lines 919-924 Link Here
919	http_cmd(conn, "User-Agent: %s " _LIBFETCH_VER, getprogname());	941	http_cmd(conn, "User-Agent: %s " _LIBFETCH_VER, getprogname());
920	if (url->offset > 0)	942	if (url->offset > 0)
921	http_cmd(conn, "Range: bytes=%lld-", (long long)url->offset);	943	http_cmd(conn, "Range: bytes=%lld-", (long long)url->offset);
		944	http_cmd(conn, "Accept-Encoding: gzip,deflate");
922	http_cmd(conn, "Connection: close");	945	http_cmd(conn, "Connection: close");
923	http_cmd(conn, "");	946	http_cmd(conn, "");
924		947
Lines 999-1004 Link Here
999	case hdr_error:	1022	case hdr_error:
1000	http_seterr(HTTP_PROTOCOL_ERROR);	1023	http_seterr(HTTP_PROTOCOL_ERROR);
1001	goto ouch;	1024	goto ouch;
		1025	case hdr_content_encoding:
		1026	encoding = http_parse_encoding(p);
		1027	break;
1002	case hdr_content_length:	1028	case hdr_content_length:
1003	http_parse_length(p, &clength);	1029	http_parse_length(p, &clength);
1004	break;	1030	break;
Lines 1119-1125 Link Here
1119		1145
1120	/* fill in stats */	1146	/* fill in stats */
1121	if (us) {	1147	if (us) {
1122	us->size = size;	1148	/* we can only predict the size of unencoded streams */
		1149	if (encoding == ENCODING_RAW)
		1150	us->size = size;
1123	us->atime = us->mtime = mtime;	1151	us->atime = us->mtime = mtime;
1124	}	1152	}
1125		1153
Lines 1139-1144 Link Here
1139	goto ouch;	1167	goto ouch;
1140	}	1168	}
1141		1169
		1170	/* wrap the decoder around it */
		1171	if ((d = httpDecode(f, encoding, size)) == NULL) {
		1172	fetch_syserr();
		1173	fclose(f);
		1174	goto ouch;
		1175	}
		1176
1142	if (url != URL)	1177	if (url != URL)
1143	fetchFreeURL(url);	1178	fetchFreeURL(url);
1144	if (purl)	1179	if (purl)
Lines 1150-1156 Link Here
1150	f = NULL;	1185	f = NULL;
1151	}	1186	}
1152		1187
1153	return (f);	1188	return (d);
1154		1189
1155	ouch:	1190	ouch:
1156	if (url != URL)	1191	if (url != URL)

Line 0 Link Here

(-)src/lib/libfetch/httpdecode.c (+411 lines)
		1	/*
		2	* I wrote this and I say you can do whatever you want with it. Period.
		3	* However, I'd love to hear from you what you've done.
		4	*
		5	* Dominic Fandrey <kamikaze@bsdforen.de>
		6	*/
		7
		8	/**
		9	* \file httpdecode.c
		10	*
		11	* This file contains the implemention of the prototypes defined in
		12	* httpdecode.h.
		13	*
		14	* @brief
		15	* HTTP content decoding implemention.
		16	* @see
		17	* httpdecode.h
		18	* @author
		19	* Dominic Fandrey <kamikaze@bsdforen.de>
		20	* @version
		21	* 0.1.99.2008.07.07
		22	*/
		23
		24	/* LINTLIBRARY */
		25
		26	#include <stdio.h>
		27	#include <stdlib.h>
		28	#include <string.h>
		29	#include <unistd.h>
		30	#include <errno.h>
		31	#include <zlib.h>
		32	#include "httpdecode.h"
		33
		34	/* PRIVATE STRUCTS */
		35
		36	/**
		37	* @brief
		38	* The necessary data to maintain a zlib decoding stream.
		39	*/
		40	struct zlibStream {
		41	/**
		42	* @brief
		43	* The original stream with the encoded data.
		44	*/
		45	FILE * source;
		46
		47	/**
		48	* @brief
		49	* A read buffer for the encoded stream.
		50	*/
		51	char * buffer;
		52
		53	/**
		54	* This specifies the encoding of the data. The values
		55	* ENCODING_GZIP and ENCODING_DEFLATE are possible.
		56	*
		57	* @brief
		58	* The encoding type of the stream.
		59	*/
		60	int encoding;
		61
		62	/**
		63	* @brief
		64	* The stream data used by zlib.
65	*/
66	z_stream stream;
67
68	/**
69	* The length of the source stream. The value 0 means that the length
70	* is unknown higher values will be used to automatically close the
71	* stream. This prevents overreading and allows the continued use
72	* of the underlying HTTP stream.
73	*
74	* @brief
75	* The length of the encoded source stream.
76	*/
77	size_t length;
78
79	/**
80	* @brief
81	* The amount of data that has been read.
82	*/
83	size_t read;
84
85	/**
86	* @brief
87	* The size of the buffer for encoded data.
88	*/
89	size_t bufferSize;
90
91	/**
92	* @brief
93	* The amount of data available in the buffer.
94	*/
95	size_t bufferUsed;
96	};
97
98	/* PRIVATE PROTOTYPES */
99	void moveBuffer(struct zlibStream * cookie, char * newBuffer, size_t size);
100	FILE * zlibOpen(struct zlibStream * cookie);
101	size_t zlibRead(struct zlibStream * cookie, char * buffer, size_t length);
102	int zlibClose(struct zlibStream * cookie);
103	/* TODO
104	FILE * compressOpen(struct zlibStream * cookie);
105	size_t compressRead(struct zlibStream * cookie, char * buffer, size_t length);
106	int compressClose(struct zlibStream * cookie);
107	FILE * randomOpen(struct zlibStream * cookie);
108	size_t randomRead(struct zlibStream * cookie, char * buffer, size_t length);
109	int randomSeek(struct zlibStream * cookie, off_t offset, int whence);
110	int randomClose(struct zlibStream * cookie);
111	*/
112
113	/* PUBLIC FUNCTIONS */
114
115	/**
116	* Opens a given stream for decoding and returns a FILE handle that can be
117	* used with the fread and fclose function. Internally funopen is used
118	* to achieve this.
119	*
120	* In case of failure NULL is returned and errno is set to EINVAL for
121	* invalid parameters and ENOMEM for insufficient memory.
122	*
123	* @brief
124	* Open a FILE stream to read decoded data from.
125	* @param source
126	* The stream to read the encoded data from.
127	* @param encoding
128	* The encoding type of the source stream.
129	* @param length
130	* The length of the source stream. Use 0 if unknown.
131	* @return
132	* Returns a FILE handle to read an encoded stream.
133	* @see
134	* funopen(3)
135	* @see
136	* fread(3)
137	* @see
138	* fclose(3)
139	*/
140	FILE * httpDecode(FILE * source, int encoding, size_t length) {
141	struct zlibStream * zlibCookie;
142
143	switch (encoding) {
144	case ENCODING_RAW:
145	return(source);
146	case ENCODING_GZIP: case ENCODING_DEFLATE:
147	zlibCookie = malloc(sizeof(struct zlibStream));
148	if (zlibCookie == NULL) /* errno == ENOMEM */
149	return(NULL);
150	zlibCookie->buffer = NULL;
151	zlibCookie->bufferSize = 0;
152	zlibCookie->source = source;
153	zlibCookie->length = length;
154	zlibCookie->read = 0;
155	zlibCookie->encoding = encoding;
156	return(zlibOpen(zlibCookie));
157	case ENCODING_COMPRESS:
158	return(NULL);
159	}
160
161	return(NULL);
162	}
163
164	/**
165	* This function is a wraper around httpDecode that allows random access
166	* by writing the stream into a temporary file. The file is buffered
167	* by a given number of buffers in memory.
168	* Buffers are overwritten in LRU order.
169	*
170	* @brief
171	* A file backed wrapper around httpDecode for random access.
172	* @param source
173	* The stream to read the encoded data from.
174	* @param encoding
175	* The encoding type of the source stream.
176	* @param length
177	* The length of the source stream. Use 0 if unknown.
178	* @param bufferSize
179	* The size of a buffer.
180	* @param
181	* The number of buffers.
182	* @return
183	* Returns a FILE handle to read an encoded stream.
184	*/
185	/* TODO
186	FILE * httpDecodeRandom(FILE * source, int encoding, size_t length,
187	size_t bufferSize, size_t buffers) {
188	return(source);
189	}
190	*/
191
192	/* PRIVATE FUNCTIONS */
193
194	/**
195	* This function replaces the read buffer in the cookie with the new buffer.
196	* The old buffer is freed but the contents are saved in the new buffer.
197	* However, no security checks are performed.
198	* That means that newBuffer must at least have the same size as the old one.
199	*
200	* @brief
201	* Replace the current read buffer.
202	* @param cookie
203	* Contains all the data necessary to maintain the stream.
204	* @param newBuffer
205	* The new buffer to use.
206	*/
207	void moveBuffer(struct zlibStream * cookie, char * newBuffer, size_t size) {
208	memmove(newBuffer, cookie->buffer, cookie->bufferUsed);
209	free(cookie->buffer);
210	cookie->buffer = newBuffer;
211	cookie->bufferSize = size;
212	}
213
214	/**
215	* This function initializes a zlib stream and creates the file handler
216	* that will later be used to pull data from the stream.
217	*
218	* Upon any kind of failure errno is set to one of the following values:
219	* EINVAL This can either indicate that an unsupported encoding
220	* was given or that this code and the used zlib implemention
221	* are incompatible.
222	* ENOMEM Indicates that the available memory is insuficient for
223	* the decode buffer, zlib or funopen.
224	*
225	* @brief
226	* Open a zlib stream.
227	* @param cookie
228	* Contains all the data necessary to maintain the stream.
229	* @return
230	* A FILE* pointer or NULL in case of failure.
231	*/
232	FILE * zlibOpen(struct zlibStream * cookie) {
233	int wbits;
234	z_stream * stream = &(cookie->stream);
235
236	/* Initialize zlib stream data. */
237	stream->zalloc = Z_NULL;
238	stream->zfree = Z_NULL;
239	stream->opaque = Z_NULL;
240	stream->avail_in = 0;
241	stream->next_in = (Bytef *) cookie->buffer;
242
243	/* Set window bits for the selected encoding. */
244	switch(cookie->encoding) {
245	case ENCODING_DEFLATE:
246	wbits = -MAX_WBITS;
247	break;
248	case ENCODING_GZIP:
249	wbits = MAX_WBITS + 16;
250	break;
251	default:
252	errno = EINVAL;
253	return(NULL);
254	}
255
256	/* Create the decoding buffer. */
257	cookie->bufferSize = 32768; /* Zlib breaks with a smaller buffer. */
258	cookie->bufferUsed = 0;
259	cookie->buffer = malloc(cookie->bufferSize);
260	if (cookie->buffer == NULL)
261	return(NULL); /* errno == ENOMEM */
262
263	/* Initialize stream for decoding. */
264	switch(inflateInit2(stream, wbits)) {
265	case Z_OK:
266	errno = 0;
267	break;
268	case Z_MEM_ERROR:
269	errno = ENOMEM;
270	break;
271	case Z_STREAM_ERROR: /* This is not supposed to happen. */
272	errno = EINVAL;
273	break;
274	}
275	if (errno) {
276	free(cookie->buffer);
277	return(NULL);
278	}
279
280	/* Create the file stream to return. */
281	return(funopen(cookie,(int ()(void , char *, int)) zlibRead,
282	NULL, NULL, (int ()(void )) zlibClose));
283	}
284
285	/**
286	* Writes a chunk of decoded data to the given buffer.
287	*
288	* In case of an error (size_t) -1 is returned to indicate to the funopen
289	* wrapper that an error occured. In such a case errno is set to EIO.
290	*
291	* An error does not cause the stream to be closed.
292	*
293	* @brief
294	* Read decoded data from the encoded stream.
295	* @param cookie
296	* Contains all the data necessary to maintain the stream.
297	* @param buffer
298	* The buffer to write the decoded data to.
299	* @param length
300	* The space available in the buffer.
301	* @return
302	* The number of bytes written to the buffer or (size_t) -1 in case of
303	* failure.
304	*/
305	size_t zlibRead(struct zlibStream * cookie, char * buffer, size_t length) {
306	char * tmpBuffer;
307	size_t growth, maxRead, bufferAvailable, flushed;
308	int zlibStatus;
309	z_stream * stream = &(cookie->stream);
310
311	/*
312	* Adjust buffer size if the target buffer is larger than 2 times
313	* the source buffer.
314	*/
315	if ((length >> 1) > cookie->bufferSize) {
316	tmpBuffer = malloc(length >> 1);
317
318	/*
319	* If creating a new buffer fails pretend never to have
320	* attempted it.
321	*/
322	if (tmpBuffer == NULL)
323	errno = 0;
324	else
325	/* Move data from the old buffer to the new one. */
326	moveBuffer(cookie, tmpBuffer, length >> 1);
327	}
328
329	/* Run until the target buffer has been filled. */
330	flushed = 0;
331	while (length > 0) {
332	/* If the input buffer is not full, fill it. */
333	growth = 0;
334	maxRead = cookie->length - cookie->read;
335	bufferAvailable = cookie->bufferSize - cookie->bufferUsed;
336	bufferAvailable = (maxRead < bufferAvailable \
337	? maxRead : bufferAvailable);
338	if (bufferAvailable > 0) {
339	growth = fread(cookie->buffer, sizeof(char), \
340	bufferAvailable, cookie->source);
341	/* Forward errors. */
342	if (ferror(cookie->source))
343	return((size_t) -1);
344	cookie->bufferUsed += growth;
345	cookie->read += growth;
346	}
347
348	/* Decode data from the read to the target buffer. */
349	stream->avail_in = cookie->bufferUsed;
350	stream->next_in = (Bytef *) cookie->buffer;
351	stream->avail_out = length;
352	stream->next_out = (Bytef *) buffer;
353	zlibStatus = inflate(stream, Z_SYNC_FLUSH);
354
355	/* The amount of data just written to the target buffer. */
356	growth = length - stream->avail_out;
357
358	/* Adjust the read buffer. */
359	memmove(cookie->buffer, stream->next_in, \
360	(size_t) stream->avail_in);
361	cookie->bufferUsed = stream->avail_in;
362	stream->next_in = (Bytef *) cookie->buffer;
363
364	/* Adjust the target buffer. */
365	flushed += growth;
366	buffer += growth;
367	length = stream->avail_out;
368
369	/* Deal with errors. */
370	switch (zlibStatus) {
371	case Z_OK:
372	break;
373	case Z_STREAM_END:
374	length = 0;
375	break;
376	case Z_BUF_ERROR:
377	/* The read buffer is too small, try to double it. */
378	tmpBuffer = malloc(cookie->bufferSize << 1);
379	if (!tmpBuffer) /* errno == ENOMEN */
380	return((size_t) -1);
381	moveBuffer(cookie, tmpBuffer, cookie->bufferSize << 1);
382	break;
383	case Z_NEED_DICT: case Z_DATA_ERROR: case Z_STREAM_ERROR:
384	errno = EIO;
385	return((size_t) -1);
386	case Z_MEM_ERROR:
387	errno = ENOMEM;
388	return((size_t) -1);
389	}
390	}
391
392	return(flushed);
393	}
394
395	/**
396	* Closes the decoding stream and frees all buffers.
397	*
398	* @brief
399	* Closes the decoding stream.
400	* @param cookie
401	* Contains all the data necessary to maintain the stream.
402	* @return
403	* Always 0 for success.
404	*/
405	int zlibClose(struct zlibStream * cookie) {
406	inflateEnd(&(cookie->stream));
407	free(cookie->buffer);
408	free(cookie);
409	return(0);
410	}
411

Line 0 Link Here

(-)src/lib/libfetch/httpdecode.h (+116 lines)
		1	/*
		2	* I wrote this and I say you can do whatever you want with it. Period.
		3	* However, I'd love to hear from you what you've done.
		4	*
		5	* Dominic Fandrey <kamikaze@bsdforen.de>
		6	*/
		7
		8	#ifndef HTTPDECODE_H
		9	#define HTTPDECODE_H
		10
		11	/**
		12	* \file httpdecode.h
		13	*
		14	* This file contains the public prototypes and defines required to read
		15	* compressed data streams. Supported formats are those listed in
		16	* RFC2616 section 3.5 (HTTP 1.1 content encodings). Compress decoding is
		17	* not yet implemented.
		18	*
		19	* @brief
		20	* Public defines and prototypes to decode encoded HTML streams.
		21	* @see
		22	* http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.5
		23	* @see
		24	* httpdecode.c
		25	* @author
		26	* Dominic Fandrey <kamikaze@bsdforen.de>
		27	* @version
		28	* 0.3.99.2008.07.07
		29	*/
		30
		31	/**
		32	* This can be used to abuse httpDecodeRandom as a random (read) access layer
		33	* for any FILE stream.
		34	*
		35	* @brief
		36	* The source stream ist not encoded.
		37	*/
		38	#define ENCODING_RAW 0
		39
		40	/**
		41	* @brief
		42	* The source stream is deflate encoded.
		43	* @see
		44	* zlib(3)
		45	*/
		46	#define ENCODING_DEFLATE 1
		47
		48	/**
		49	* @brief
		50	* The source stream is gzip encoded.
		51	* @see
		52	* gzip(1)
		53	* @see
		54	* zlib(3)
		55	*/
		56	#define ENCODING_GZIP 2
		57
		58	/**
		59	* @brief
		60	* The source stream is compress encoded.
		61	* @see
		62	* compress(1)
		63	*/
		64	#define ENCODING_COMPRESS 3
65
66
67	/**
68	* Opens a given stream for decoding and returns a FILE handle that can be
69	* used with the read and close function. Internally funopen is used
70	* to achieve this.
71	*
72	* @param source
73	* The stream to read the encoded data from.
74	* @param encoding
75	* The encoding type of the source stream.
76	* @param length
77	* The length of the source stream. Use 0 if unknown.
78	* @return
79	* Returns a FILE handle to read an encoded stream.
80	* @see
81	* funopen(3)
82	* @see
83	* fread(3)
84	* @see
85	* fclose(3)
86	*/
87	FILE * httpDecode(FILE * source, int encoding, size_t length);
88
89	/**
90	* This function is a wraper around httpDecode that allows random access
91	* by writing the stream into a temporary file. The file is buffered
92	* by a given number of buffers in memory.
93	* Buffers are overwritten in LRU order.
94	*
95	* @brief
96	* A file backed wrapper around httpDecode for random access.
97	* @param source
98	* The stream to read the encoded data from.
99	* @param encoding
100	* The encoding type of the source stream.
101	* @param length
102	* The length of the source stream. Use 0 if unknown.
103	* @param bufferSize
104	* The size of a buffer.
105	* @param
106	* The number of buffers.
107	* @return
108	* Returns a FILE handle to read an encoded stream.
109	*/
110	/* TODO
111	FILE * httpDecodeRandom(FILE * source, int encoding, size_t length,
112	size_t bufferSize, size_t buffers);
113	*/
114
115	#endif /* HTTPDECODE_H */
116