[distcc] LZO compression (updated)
Stephen White
swhite at decisionsoft.com
Wed Jan 29 11:51:01 GMT 2003
I've had a little more time to update the LZO compression patch that I
create a couple of months ago.
* LZO compression is optional, to enable use DISTCC_COMPRESS=1
When this isn't set (or it's set to DISTCC_COMPRESS=0) the client
should interoperate with older distcc servers that do not support
compression.
* The server should work equally well with clients that do not support
compression.
* Configure should use the system installed liblzo (if it exists) .. if
not then it will use it's own copy.
* Compression is now done in blocks, rather than loading the whole
file into RAM. Currently this block size is set to 64K, but this
can be configured in bulk_lzo.h
* The code has been tidied up a fair bit.
Still on the TODO list:
* Documentation!
* Possible inclusion of message digests in the protocol to ensure
data is uncorrupted.
--
Stephen White +44-1865-203192
DecisionSoft Limited http://www.decisionsoft.com
XML Development and Services
-------------- next part --------------
diff -N -aur distcc.clean/Makefile.in distcc.my.clean/Makefile.in
--- distcc.clean/Makefile.in Wed Jan 29 11:01:52 2003
+++ distcc.my.clean/Makefile.in Wed Jan 29 11:11:20 2003
@@ -80,6 +80,7 @@
$(MEN) \
$(pkgdoc_DOCS) \
$(popt_SRC) $(popt_HEADERS) \
+ $(lzo_SRC) $(lzo_HEADERS) \
$(SRC) $(HEADERS) \
$(test_SOURCE) \
$(bench_PY)
@@ -134,8 +135,8 @@
distnews = $(PACKAGE_TARNAME)-$(VERSION).NEWS
common_obj = src/trace.o src/util.o src/io.o src/exec.o src/arg.o \
-src/rpc.o src/tempfile.o src/bulk.o src/help.o src/filename.o \
-src/lock.o src/where.o src/hosts.o src/sendfile.o
+ src/rpc.o src/tempfile.o src/bulk.o src/bulk_lzo.o src/help.o \
+ src/filename.o src/lock.o src/where.o src/hosts.o src/sendfile.o @BUILD_LZO@
distcc_obj = src/cpp.o src/distcc.o src/clinet.o src/strip.o \
src/implicit.o $(common_obj)
@@ -152,7 +153,7 @@
h_strip_obj = src/h_strip.o src/trace.o src/util.o src/arg.o src/filename.o src/strip.o
h_parsemask_obj = src/h_parsemask.o $(common_obj) src/access.o
-SRC = src/arg.c src/bulk.c src/clinet.c src/cpp.c \
+SRC = src/arg.c src/bulk.c src/clinet.c src/cpp.c, src/bulk_lzo.c \
src/daemon.c src/distcc.c src/sendfile.c src/access.c \
src/dopt.c src/dparent.c src/exec.c src/filename.c \
src/h_argvtostr.c \
@@ -165,12 +166,18 @@
HEADERS = src/access.h src/clinet.h src/cpp.h \
src/distcc.h src/exitcode.h src/hosts.h \
src/io.h src/tempfile.h src/setuid.h \
- src/strip.h src/implicit.h \
+ src/strip.h src/implicit.h src/bulk_lzo.h src/bulk.h \
src/opt.h src/rpc.h src/trace.h src/util.h src/bulk.h \
src/exec.h src/lock.h src/where.h src/srvnet.h
MEN = man/distcc.1 man/distccd.1
+lzo_OBJS=lzo/minilzo.o
+
+lzo_HEADERS=lzo/minilzo.h lzo/lzoconf.h
+
+lzo_SRC=lzo/minilzo.c lzo/testmini.c
+
popt_OBJS=popt/findme.o popt/popt.o popt/poptconfig.o \
popt/popthelp.o popt/poptparse.o
@@ -256,9 +263,6 @@
h_strip: $(h_strip_obj)
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)
-
-
-
## DIST targets
# The sub-targets copy (and if necessary, build) various files that
@@ -341,6 +345,7 @@
clean:
rm -f src/*.o popt/*.o
rm -f $(check_PROGRAMS) $(bin_PROGRAMS)
+ make -C lzo clean
rm -rf testtmp
maintainer-clean: distclean maintainer-clean-web maintainer-clean-linuxdoc \
diff -N -aur distcc.clean/configure.ac distcc.my.clean/configure.ac
--- distcc.clean/configure.ac Wed Jan 29 11:01:52 2003
+++ distcc.my.clean/configure.ac Wed Jan 29 11:11:26 2003
@@ -52,7 +52,11 @@
### Checks for configure options
AC_ARG_WITH(included-popt,
- AC_HELP_STRING([--with-included-popt], [use bundled popt library, not from system]))
+ [ --with-included-popt use bundled popt library, not from system])
+
+AC_ARG_WITH(included-lzo,
+ [ --with-included-lzo use bundled lzo library, not shared system library])
+# AC_HELP_STRING([--with-included-popt], [use bundled popt library, not from system]))
dnl Checks for programs
AC_PROG_CC
@@ -115,6 +119,7 @@
########################################################################
### Checks for libraries.
+
AC_CHECK_LIB(nsl, gethostbyname)
AC_CHECK_LIB(socket, connect)
AC_CHECK_LIB(resolv, hstrerror, , , [-lnsl -lsocket])
@@ -137,8 +142,24 @@
AC_MSG_RESULT(no)
fi
-########################################################################
-# Checks for library functions, using libraries discovered above
+if test x"$with_included_lzo" != x"yes" && test x"$with_included_lzo" != xno
+then
+ # If not explicitly requested, guess
+ AC_CHECK_LIB(lzo, lzo1x_1_compress, , [with_included_lzo=yes])
+ AC_CHECK_LIB(lzo, lzo1x_decompress_safe, , [with_included_lzo=yes])
+fi
+AC_CACHE_SAVE
+
+AC_MSG_CHECKING([whether to use included minilzo])
+if test x"$with_included_lzo" = x"yes"
+then
+ AC_MSG_RESULT($srcdir/lzo)
+ BUILD_LZO='$(lzo_OBJS)'
+ CPPFLAGS="$CPPFLAGS -I$srcdir/lzo"
+else
+ AC_MSG_RESULT(no)
+fi
+
CPPFLAGS="$CPPFLAGS -I$srcdir/src"
AC_CHECK_FUNCS([sendfile setsid flock lockf hstrerror strerror setuid setreuid])
@@ -163,6 +184,7 @@
AC_SUBST(LDFLAGS)
AC_SUBST(CPPFLAGS)
AC_SUBST(BUILD_POPT)
+AC_SUBST(BUILD_LZO)
dnl AC_DEFINE_UNQUOTED(PACKAGE, $PACKAGE, [Package name])
dnl AC_DEFINE_UNQUOTED(VERSION, $VERSION, [Package version])
AC_DEFINE_UNQUOTED(GNU_HOST, ["$host"], [Your gnu-style host triple])
diff -N -aur distcc.clean/lzo/README.LZO distcc.my.clean/lzo/README.LZO
--- distcc.clean/lzo/README.LZO Mon Dec 16 03:43:42 2002
+++ distcc.my.clean/lzo/README.LZO Wed Jan 29 11:11:34 2003
@@ -1,3 +1,12 @@
+This is a copy of the miniLZO distribution, it is included with distcc to ease compilation on systems that do not have the full lzo library installed. On systems with an lzo library available this will not be used.
+
+The only modification to this distribution is to rename 'minilzo.h' to 'lzo1x.h' so that the code can use the same '#include' directives regardless of which distribution is in use.
+
+The remainder of this file is the standard miniLZO readme.
+
+-- Stephen White (2003/1/24)
+
+
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
diff -N -aur distcc.clean/lzo/lzo1x.h distcc.my.clean/lzo/lzo1x.h
--- distcc.clean/lzo/lzo1x.h Thu Jan 1 01:00:00 1970
+++ distcc.my.clean/lzo/lzo1x.h Wed Jan 29 11:10:58 2003
@@ -0,0 +1,100 @@
+/* minilzo.h -- mini subset of the LZO real-time data compression library
+
+ This file is part of the LZO real-time data compression library.
+
+ Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer
+ Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer
+ Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer
+ Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer
+ Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer
+ Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer
+ Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer
+ All Rights Reserved.
+
+ The LZO library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU General Public License as
+ published by the Free Software Foundation; either version 2 of
+ the License, or (at your option) any later version.
+
+ The LZO library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with the LZO library; see the file COPYING.
+ If not, write to the Free Software Foundation, Inc.,
+ 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+ Markus F.X.J. Oberhumer
+ <markus at oberhumer.com>
+ http://www.oberhumer.com/opensource/lzo/
+ */
+
+/*
+ * NOTE:
+ * the full LZO package can be found at
+ * http://www.oberhumer.com/opensource/lzo/
+ */
+
+
+#ifndef __MINILZO_H
+#define __MINILZO_H
+
+#define MINILZO_VERSION 0x1080
+
+#ifdef __LZOCONF_H
+# error "you cannot use both LZO and miniLZO"
+#endif
+
+#undef LZO_HAVE_CONFIG_H
+#include "lzoconf.h"
+
+#if !defined(LZO_VERSION) || (LZO_VERSION != MINILZO_VERSION)
+# error "version mismatch in header files"
+#endif
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/***********************************************************************
+//
+************************************************************************/
+
+/* Memory required for the wrkmem parameter.
+ * When the required size is 0, you can also pass a NULL pointer.
+ */
+
+#define LZO1X_MEM_COMPRESS LZO1X_1_MEM_COMPRESS
+#define LZO1X_1_MEM_COMPRESS ((lzo_uint32) (16384L * lzo_sizeof_dict_t))
+#define LZO1X_MEM_DECOMPRESS (0)
+
+
+/* compression */
+LZO_EXTERN(int)
+lzo1x_1_compress ( const lzo_byte *src, lzo_uint src_len,
+ lzo_byte *dst, lzo_uintp dst_len,
+ lzo_voidp wrkmem );
+
+/* decompression */
+LZO_EXTERN(int)
+lzo1x_decompress ( const lzo_byte *src, lzo_uint src_len,
+ lzo_byte *dst, lzo_uintp dst_len,
+ lzo_voidp wrkmem /* NOT USED */ );
+
+/* safe decompression with overrun testing */
+LZO_EXTERN(int)
+lzo1x_decompress_safe ( const lzo_byte *src, lzo_uint src_len,
+ lzo_byte *dst, lzo_uintp dst_len,
+ lzo_voidp wrkmem /* NOT USED */ );
+
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif /* already included */
+
diff -N -aur distcc.clean/lzo/minilzo.h distcc.my.clean/lzo/minilzo.h
--- distcc.clean/lzo/minilzo.h Mon Dec 16 03:43:42 2002
+++ distcc.my.clean/lzo/minilzo.h Thu Jan 1 01:00:00 1970
@@ -1,100 +0,0 @@
-/* minilzo.h -- mini subset of the LZO real-time data compression library
-
- This file is part of the LZO real-time data compression library.
-
- Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer
- Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer
- Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer
- Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer
- Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer
- Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer
- Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer
- All Rights Reserved.
-
- The LZO library is free software; you can redistribute it and/or
- modify it under the terms of the GNU General Public License as
- published by the Free Software Foundation; either version 2 of
- the License, or (at your option) any later version.
-
- The LZO library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with the LZO library; see the file COPYING.
- If not, write to the Free Software Foundation, Inc.,
- 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-
- Markus F.X.J. Oberhumer
- <markus at oberhumer.com>
- http://www.oberhumer.com/opensource/lzo/
- */
-
-/*
- * NOTE:
- * the full LZO package can be found at
- * http://www.oberhumer.com/opensource/lzo/
- */
-
-
-#ifndef __MINILZO_H
-#define __MINILZO_H
-
-#define MINILZO_VERSION 0x1080
-
-#ifdef __LZOCONF_H
-# error "you cannot use both LZO and miniLZO"
-#endif
-
-#undef LZO_HAVE_CONFIG_H
-#include "lzoconf.h"
-
-#if !defined(LZO_VERSION) || (LZO_VERSION != MINILZO_VERSION)
-# error "version mismatch in header files"
-#endif
-
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-
-/***********************************************************************
-//
-************************************************************************/
-
-/* Memory required for the wrkmem parameter.
- * When the required size is 0, you can also pass a NULL pointer.
- */
-
-#define LZO1X_MEM_COMPRESS LZO1X_1_MEM_COMPRESS
-#define LZO1X_1_MEM_COMPRESS ((lzo_uint32) (16384L * lzo_sizeof_dict_t))
-#define LZO1X_MEM_DECOMPRESS (0)
-
-
-/* compression */
-LZO_EXTERN(int)
-lzo1x_1_compress ( const lzo_byte *src, lzo_uint src_len,
- lzo_byte *dst, lzo_uintp dst_len,
- lzo_voidp wrkmem );
-
-/* decompression */
-LZO_EXTERN(int)
-lzo1x_decompress ( const lzo_byte *src, lzo_uint src_len,
- lzo_byte *dst, lzo_uintp dst_len,
- lzo_voidp wrkmem /* NOT USED */ );
-
-/* safe decompression with overrun testing */
-LZO_EXTERN(int)
-lzo1x_decompress_safe ( const lzo_byte *src, lzo_uint src_len,
- lzo_byte *dst, lzo_uintp dst_len,
- lzo_voidp wrkmem /* NOT USED */ );
-
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif
-
-#endif /* already included */
-
diff -N -aur distcc.clean/src/bulk.c distcc.my.clean/src/bulk.c
--- distcc.clean/src/bulk.c Thu Dec 12 12:14:41 2002
+++ distcc.my.clean/src/bulk.c Wed Jan 29 11:11:40 2003
@@ -36,7 +36,6 @@
* keep writing), but we can't send from a fifo, because we wouldn't
* know how many bytes were coming.
*
- * @todo Optionally support lzo compression
**/
#include "config.h"
@@ -56,6 +55,7 @@
#include "io.h"
#include "rpc.h"
#include "bulk.h"
+#include "bulk_lzo.h"
/**
* Open a file for read, and also put its size into @p fsize.
@@ -114,7 +114,7 @@
* transmitted.
**/
int dcc_x_file(int ofd, const char *fname, const char *token,
- size_t *size_out)
+ size_t *size_out, int use_lzo)
{
int ifd;
off_t f_size;
@@ -132,6 +132,7 @@
goto failed;
if (f_size) {
+ if (!use_lzo) {
#ifdef HAVE_SENDFILE
if (dcc_pump_sendfile(ofd, ifd, (size_t) f_size))
goto failed;
@@ -139,6 +140,10 @@
if (dcc_pump_readwrite(ofd, ifd, (size_t) f_size))
goto failed;
#endif
+ } else {
+ if (dcc_r_file_body_pack(ofd,ifd,(size_t) f_size))
+ goto failed;
+ }
}
if (ifd != -1)
@@ -160,7 +165,7 @@
*
* @param filename local filename to create.
**/
-int dcc_r_file(int ifd, const char *filename, size_t len)
+int dcc_r_file(int ifd, const char *filename, size_t len, int use_lzo)
{
int ofd;
int ret, close_ret;
@@ -176,7 +181,7 @@
return -1;
}
- ret = dcc_r_file_body(ofd, ifd, len);
+ ret = dcc_r_file_body(ofd, ifd, len, use_lzo);
close_ret = dcc_close(ofd);
if (!ret && !close_ret) {
@@ -200,7 +205,7 @@
* The fifo is closed when we either reach the end of the network
* body, or fail.
**/
-int dcc_r_fifo(int ifd, const char *fifo_name, size_t len)
+int dcc_r_fifo(int ifd, const char *fifo_name, size_t len, int use_lzo)
{
int ofd, ret, close_ret;
@@ -218,7 +223,7 @@
/* Removing the fifo wouldn't help because the child has already
* started reading */
- ret = dcc_r_file_body(ofd, ifd, len);
+ ret = dcc_r_file_body(ofd, ifd, len, use_lzo);
close_ret = dcc_close(ofd);
if (!ret && !close_ret) {
@@ -236,7 +241,15 @@
* The output file descriptor is not closed, because this routine is
* also used for copying onto stderr.
**/
-int dcc_r_file_body(int ofd, int ifd, size_t len)
+int dcc_r_file_body(int ofd, int ifd, size_t len, int use_lzo)
{
- return dcc_pump_readwrite(ofd, ifd, (size_t) len);
+ if (len==0) {
+ return 0;
+ }
+
+ if (use_lzo) {
+ return dcc_r_file_body_unpack(ofd,ifd,len);
+ } else {
+ return dcc_pump_readwrite(ofd, ifd, (size_t) len);
+ }
}
diff -N -aur distcc.clean/src/bulk.h distcc.my.clean/src/bulk.h
--- distcc.clean/src/bulk.h Sat Aug 31 23:32:20 2002
+++ distcc.my.clean/src/bulk.h Wed Jan 29 11:11:43 2003
@@ -21,7 +21,7 @@
* USA
*/
-int dcc_r_file(int ifd, const char *filename, size_t);
-int dcc_x_file(int ofd, const char *fname, const char *token, size_t *);
-int dcc_r_file_body(int ofd, int ifd, size_t len);
-int dcc_r_fifo(int ifd, const char *fifo_name, size_t len);
+int dcc_r_file(int ifd, const char *filename, size_t, int use_lzo);
+int dcc_x_file(int ofd, const char *fname, const char *token, size_t *, int use_lzo);
+int dcc_r_file_body(int ofd, int ifd, size_t len, int use_lzo);
+int dcc_r_fifo(int ifd, const char *fifo_name, size_t len, int use_lzo);
diff -N -aur distcc.clean/src/bulk_lzo.c distcc.my.clean/src/bulk_lzo.c
--- distcc.clean/src/bulk_lzo.c Thu Jan 1 01:00:00 1970
+++ distcc.my.clean/src/bulk_lzo.c Wed Jan 29 11:11:04 2003
@@ -0,0 +1,249 @@
+/* -*- c-file-style: "java"; indent-tabs-mode: nil -*-
+ *
+ * distcc -- A simple distributed compiler system
+ * $Header: /cvsroot/distcc/src/bulk.c,v 1.23 2002/11/01 07:58:31 mbp Exp $
+ *
+ * Copyright (C) 2002 by Martin Pool <mbp at samba.org>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA
+ */
+
+/**
+ * @file
+ *
+ * Bulk file transfer, used for sending .i, .o files etc.
+ *
+ * These are the versions of the routines that carry out LZO compression,
+ * see bulk.c for more information.
+ *
+ * <stephen-distcc at decisionsoft.com>
+ **/
+
+#include "config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <assert.h>
+#include <string.h>
+#include <fcntl.h>
+#include <errno.h>
+
+#include <sys/stat.h>
+
+#include "distcc.h"
+#include "trace.h"
+#include "io.h"
+#include "rpc.h"
+
+#include <lzoconf.h>
+#include <lzo1x.h>
+
+#include "bulk_lzo.h"
+
+/**
+ * Memory for use by compression/decompression
+ */
+#define HEAP_ALLOC(var,size) \
+ lzo_align_t __LZO_MMODEL var [ ((size) + (sizeof(lzo_align_t) - 1)) / sizeof(lzo_align_t) ]
+
+#define LZO1X_1_MEM_COMPRESS ((lzo_uint32) (16384L * lzo_sizeof_dict_t))
+
+static HEAP_ALLOC(wrkmem,LZO1X_1_MEM_COMPRESS);
+
+#define min(x,y) (x>y?y:x)
+#define io_len(x) (min(SSIZE_MAX,x))
+
+/**
+ * Fills a buffer with up to 'size' bytes from fd.
+ * returns the number of bytes read
+ */
+size_t buf_from_fd(char *buf, int fd, size_t size) {
+ size_t n = 0;
+
+ while (n < size) {
+ int bytes = read(fd,buf + n, io_len(size) - n);
+
+ if (bytes <= 0) {
+ break;
+ } else {
+ n+=bytes;
+ }
+ }
+
+ return n;
+}
+
+/**
+ * Sends data from a buffer to a file descriptor
+ * returns the number of bytes sent.
+ */
+size_t buf_to_fd(char *buf, int fd, size_t size) {
+ size_t n = 0;
+ while (n < size) {
+ int bytes = write(fd,buf + n, io_len(size - n));
+
+ if (bytes < 0) {
+ break;
+ } else {
+ n+=bytes;
+ }
+ }
+
+ return n;
+}
+
+/**
+ * Like dcc_r_file_body but compress with lzo
+ */
+int dcc_r_file_body_pack(int ofd, int ifd, size_t len)
+{
+ size_t n = min(len,DISTCC_COMPRESS_BLOCK_SIZE);
+ size_t out_len = n + n / 64 + 16 + 3;
+ char *in_buf = malloc(n);
+ char *out_buf = malloc(out_len);
+
+ if ((in_buf == NULL) || (out_buf == NULL)) {
+ if (in_buf != NULL) { free(in_buf); }
+ if (out_buf != NULL) { free(out_buf); }
+ rs_log_warning("Memory allocation failed");
+ return -1;
+ }
+
+ while (len > 0) {
+ // Read maximum of DISTCC_COMPRESS_BLOCK_SIZE from file descriptor
+ // into in_buf
+ n = buf_from_fd(in_buf,ifd,min(len,DISTCC_COMPRESS_BLOCK_SIZE));
+
+ if (n <= 0) {
+ rs_log_warning("error reading from file");
+ break;
+ }
+
+ // We've read something from the file, compress it, send it to
+ // the recipient and then repeat.
+ len -= n;
+
+ // Compress
+ if (lzo1x_1_compress(in_buf,n,out_buf,&out_len,wrkmem) != LZO_E_OK) {
+ // Internal error - shouldn't happen!
+ rs_log_warning("LZO compress internal error");
+ }
+
+ // Tell the recipient how long this compressed block is,
+ // and how much data it should unpack to
+ if (dcc_x_token_int(ofd, "LZOL", out_len) ||
+ dcc_x_token_int(ofd, "OUTL", n)) {
+ rs_log_warning("Failed to send token");
+ free(in_buf);
+ free(out_buf);
+ return -1;
+ }
+
+ // Write from out_buf to output stream
+ if (buf_to_fd(out_buf,ofd,out_len) < out_len) {
+ rs_log_warning("failed to send file");
+ break;
+ }
+ }
+
+ free(in_buf);
+ free(out_buf);
+
+ if (len > 0) {
+ rs_log_warning("failed to send file (%i bytes unsent)",len);
+ return -1;
+ }
+
+ // send LZOL block of length 0 .. to indicate EOF
+ if (dcc_x_token_int(ofd, "LZOL", 0)) {
+ return -1;
+ }
+
+ rs_trace("LZO - done pack and send (%i packed bytes from %i bytes)",out_len,len);
+
+ return 0;
+}
+
+/**
+ * Like dcc_r_file_body but uncompress with lzo
+ */
+int dcc_r_file_body_unpack(int ofd, int ifd, size_t len)
+{
+ size_t in_len, n;
+ int r;
+
+ // Read compressed LZO blocks until we reach one of length 0 ..
+ // which marks the EOF
+ while (!dcc_r_token_int(ifd, "LZOL", &in_len) && in_len) {
+ char *out_buf = NULL, *in_buf = NULL;
+ size_t out_len;
+
+ if (dcc_r_token_int(ifd, "OUTL", &out_len)) {
+ rs_log_warning("Missing OUTL token");
+ return -1;
+ }
+
+ out_buf = malloc(out_len);
+ in_buf = malloc(in_len);
+
+ if ((in_buf == NULL) || (out_buf == NULL)) {
+ if (in_buf != NULL) { free(in_buf); }
+ if (out_buf != NULL) { free(out_buf); }
+ rs_log_warning("Memory allocation failed");
+ return -1;
+ }
+
+ rs_trace("reading packed stream (len %i)",in_len);
+
+ // Read in_len bytes from file descriptor into in_buf
+ if (buf_from_fd(in_buf,ifd,in_len) < in_len) {
+ rs_log_warning("error during read");
+ free(in_buf);
+ free(out_buf);
+ break;
+ }
+
+ // decompress
+ r = lzo1x_decompress_safe(in_buf,in_len,out_buf,&out_len,wrkmem);
+ if (r != LZO_E_OK) {
+ // Internal error - shouldn't happen!
+ rs_log_warning("LZO decompress internal error %i",r);
+ }
+
+ free(in_buf);
+
+ // Write from out_buf to output stream
+ n = buf_to_fd(out_buf,ofd,out_len);
+
+ if (n < out_len) {
+ rs_log_warning("error during write");
+ }
+
+ len-=n;
+
+ free(out_buf);
+ }
+
+ rs_trace("LZO decompress done");
+
+ if (len != 0) {
+ rs_log_warning("unpack fell short by %i bytes",len);
+ return -1;
+ }
+
+ return 0;
+}
diff -N -aur distcc.clean/src/bulk_lzo.h distcc.my.clean/src/bulk_lzo.h
--- distcc.clean/src/bulk_lzo.h Thu Jan 1 01:00:00 1970
+++ distcc.my.clean/src/bulk_lzo.h Wed Jan 29 11:11:07 2003
@@ -0,0 +1,29 @@
+/* -*- c-file-style: "java"; indent-tabs-mode: nil -*-
+ *
+ * distcc -- A simple distributed compiler system
+ * $Header: /cvsroot/distcc/src/bulk.h,v 1.1 2002/08/31 22:32:20 mbp Exp $
+ *
+ * Copyright (C) 2002 by Martin Pool <mbp at samba.org>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA
+ */
+
+// When sending a file, compress in blocks of 64K. Changing this value as you
+// see fit, the recieving process does not make any assumptions based on it.
+#define DISTCC_COMPRESS_BLOCK_SIZE 65536
+
+int dcc_r_file_body_pack(int ofd, int ifd, size_t len);
+int dcc_r_file_body_unpack(int ofd, int ifd, size_t len);
diff -N -aur distcc.clean/src/daemon.c distcc.my.clean/src/daemon.c
--- distcc.clean/src/daemon.c Wed Jan 29 11:01:53 2003
+++ distcc.my.clean/src/daemon.c Wed Jan 29 11:11:49 2003
@@ -60,7 +60,7 @@
#include "opt.h"
#include "setuid.h"
#include "srvnet.h"
-
+#include "lzoconf.h"
/* for trace.c */
char const *rs_program_name = "distccd";
@@ -93,6 +93,12 @@
* the right ownership. */
dccd_setup_log();
+ if (lzo_init() != LZO_E_OK)
+ {
+ printf("lzo_init() failed\n");
+ return EXIT_DISTCC_FAILED;
+ }
+
/* This test might need to be a bit more complex when we want to
* support ssh and stuff like that. Perhaps it would be better to
* always serve stdin, and then complain if it's a terminal? */
diff -N -aur distcc.clean/src/distcc.c distcc.my.clean/src/distcc.c
--- distcc.clean/src/distcc.c Wed Jan 29 11:01:53 2003
+++ distcc.my.clean/src/distcc.c Wed Jan 29 11:11:53 2003
@@ -61,7 +61,7 @@
#include "where.h"
#include "lock.h"
#include "cpp.h"
-
+#include "lzoconf.h"
/* Name of this program, for trace.c */
const char *rs_program_name = "distcc";
@@ -129,9 +129,9 @@
/*
* Transmit header for whole request.
*/
-static int dcc_x_req_header(int fd)
+static int dcc_x_req_header(int fd, int flags)
{
- return dcc_x_token_int(fd, "DIST", PROTO_VER);
+ return dcc_x_token_int(fd, "DIST", (flags << 16) | PROTO_VER);
}
@@ -161,12 +161,18 @@
char **argv,
pid_t cpp_pid,
int *status,
- const char *cpp_fname)
+ const char *cpp_fname,
+ int use_lzo)
{
long stime_usec, utime_usec;
int ret;
+ int flags = 0;
+
+ if (use_lzo) {
+ flags |= FLAG_USE_LZO;
+ }
- if ((ret = dcc_x_req_header(net_fd))
+ if ((ret = dcc_x_req_header(net_fd,flags))
|| (ret = dcc_x_argv(net_fd, argv)))
return ret;
@@ -194,7 +200,7 @@
return EXIT_IO_ERROR;
}
- if ((ret = dcc_x_file(net_fd, cpp_fname, "DOTI", NULL)))
+ if ((ret = dcc_x_file(net_fd, cpp_fname, "DOTI", NULL, use_lzo)))
return ret;
rs_trace("client finished sending request to server");
@@ -220,13 +226,14 @@
char **argv,
pid_t cpp_pid,
int *status,
- const char *cpp_fname)
+ const char *cpp_fname,
+ int use_lzo)
{
int ret;
tcp_cork_sock(net_fd, 1);
- ret = dcc_send_job(net_fd, argv, cpp_pid, status, cpp_fname);
+ ret = dcc_send_job(net_fd, argv, cpp_pid, status, cpp_fname, use_lzo);
tcp_cork_sock(net_fd, 0);
@@ -234,7 +241,7 @@
}
-static int dcc_retrieve_results(int net_fd, int *status, const char *output_fname)
+static int dcc_retrieve_results(int net_fd, int *status, const char *output_fname, int use_lzo)
{
int len;
int ret;
@@ -243,9 +250,9 @@
if ((ret = dcc_r_result_header(net_fd))
|| (ret = dcc_r_cc_status(net_fd, status))
|| (ret = dcc_r_token_int(net_fd, "SERR", &len))
- || (ret = dcc_r_file_body(STDERR_FILENO, net_fd, len))
+ || (ret = dcc_r_file_body(STDERR_FILENO, net_fd, len, use_lzo))
|| (ret = dcc_r_token_int(net_fd, "SOUT", &len))
- || (ret = dcc_r_file_body(STDOUT_FILENO, net_fd, len))
+ || (ret = dcc_r_file_body(STDOUT_FILENO, net_fd, len, use_lzo))
|| (ret = dcc_r_token_int(net_fd, "DOTO", &o_len)))
return ret;
@@ -255,7 +262,7 @@
* sends an 0 byte file on failure anyhow. */
if (o_len)
- return dcc_r_file(net_fd, output_fname, o_len);
+ return dcc_r_file(net_fd, output_fname, o_len, use_lzo);
else {
rs_log_notice("skipping retrieval of 0 byte object file %s",
output_fname);
@@ -303,6 +310,13 @@
{
int net_fd;
int ret;
+ int use_lzo = (dcc_getenv_bool("DISTCC_COMPRESS", 0));
+
+ if (use_lzo && (lzo_init() != LZO_E_OK))
+ {
+ printf("lzo_init() failed\n");
+ return 3;
+ }
*status = 0;
@@ -313,9 +327,9 @@
/* This waits for cpp and puts its status in *status. If cpp failed, then
* the connection will have been dropped and we need not bother trying to
* get any response from the server. */
- ret = dcc_send_job_corked(net_fd, argv, cpp_pid, status, cpp_fname);
+ ret = dcc_send_job_corked(net_fd, argv, cpp_pid, status, cpp_fname, use_lzo);
if (ret == 0 && *status == 0) {
- ret = dcc_retrieve_results(net_fd, status, output_fname);
+ ret = dcc_retrieve_results(net_fd, status, output_fname, use_lzo);
}
/* Close socket so that the server can terminate, rather than
diff -N -aur distcc.clean/src/distcc.h distcc.my.clean/src/distcc.h
--- distcc.clean/src/distcc.h Wed Jan 29 11:01:53 2003
+++ distcc.my.clean/src/distcc.h Wed Jan 29 11:11:56 2003
@@ -22,6 +22,8 @@
#define PROTO_VER 1
+#define FLAG_USE_LZO 0x1
+
#ifdef NORETURN
/* nothing */
#elif defined(__GNUC__)
diff -N -aur distcc.clean/src/hosts.c distcc.my.clean/src/hosts.c
--- distcc.clean/src/hosts.c Thu Dec 12 12:01:56 2002
+++ distcc.my.clean/src/hosts.c Wed Jan 29 11:12:02 2003
@@ -126,7 +126,6 @@
rs_log_warning("$DISTCC_HOSTS is not defined; can't distribute work");
return EXIT_BAD_HOSTSPEC;
}
-
return dcc_parse_hosts(where, ret_list, ret_nhosts);
}
diff -N -aur distcc.clean/src/serve.c distcc.my.clean/src/serve.c
--- distcc.clean/src/serve.c Wed Jan 29 11:01:54 2003
+++ distcc.my.clean/src/serve.c Wed Jan 29 11:12:08 2003
@@ -83,8 +83,10 @@
#include "exec.h"
#include "srvnet.h"
-
-static int dcc_r_request_header(int ifd)
+/**
+ * Check the DIST protocol version header and read any flags.
+ */
+static int dcc_r_request_header(int ifd, int *flags)
{
int vers;
@@ -93,7 +95,9 @@
return -1;
}
- if (vers != PROTO_VER) {
+ *flags = vers >> 16;
+
+ if ((vers & 0xffff) != PROTO_VER) {
rs_log_error("client version is %d, i am %d", vers, 1);
return -1;
}
@@ -214,7 +218,7 @@
**/
static int dcc_server_compile(char **argv, int netfd,
const char *temp_i,
- const char *temp_out, const char *temp_err)
+ const char *temp_out, const char *temp_err, int flags)
{
int i_size;
int ret;
@@ -228,7 +232,7 @@
rs_log_info("input file is %d bytes", i_size);
/* read input before spawning */
- if ((ret = dcc_r_file(netfd, temp_i, i_size)) != 0)
+ if ((ret = dcc_r_file(netfd, temp_i, i_size, flags & FLAG_USE_LZO)) != 0)
return ret;
return dcc_spawn_child("compiler", argv, &pid, "/dev/null", temp_out, temp_err);
@@ -279,6 +283,7 @@
char *input_exten;
long u_us, s_us;
int i_size;
+ int flags;
pid_t cc_pid;
err_fname = dcc_make_tmpnam("cc", ".err");
@@ -311,7 +316,7 @@
/* Allow output to accumulate into big packets. */
tcp_cork_sock(netfd, 1);
- if ((ret = dcc_r_request_header(netfd))
+ if ((ret = dcc_r_request_header(netfd,&flags))
|| (ret = dcc_r_argv(netfd, &argv))
|| (ret = dcc_scan_args(argv, &orig_input, &orig_output, &argv)))
return ret;
@@ -331,7 +336,7 @@
temp_o = dcc_make_tmpnam("server", ".out");
if ((ret = dcc_r_token_int(netfd, "DOTI", &i_size))
- || (ret = dcc_r_file(netfd, temp_i, (size_t) i_size))
+ || (ret = dcc_r_file(netfd, temp_i, (size_t) i_size, flags & FLAG_USE_LZO))
|| (ret = dcc_set_input(argv, temp_i))
|| (ret = dcc_set_output(argv, temp_o)))
return ret;
@@ -344,13 +349,13 @@
if ((ret = dcc_x_result_header(netfd))
|| (ret = dcc_x_cc_status(netfd, status))
- || (ret = dcc_x_file(netfd, err_fname, "SERR", NULL))
- || (ret = dcc_x_file(netfd, out_fname, "SOUT", NULL))
+ || (ret = dcc_x_file(netfd, err_fname, "SERR", NULL, flags & FLAG_USE_LZO))
+ || (ret = dcc_x_file(netfd, out_fname, "SOUT", NULL, flags & FLAG_USE_LZO))
|| WIFSIGNALED(status)
|| WEXITSTATUS(status)) {
dcc_x_token_int(netfd, "DOTO", 0);
} else {
- ret = dcc_x_file(netfd, temp_o, "DOTO", &o_size);
+ ret = dcc_x_file(netfd, temp_o, "DOTO", &o_size, flags & FLAG_USE_LZO);
}
dcc_report_rusage(argv[0], u_us, s_us);
More information about the distcc
mailing list