[distcc] Updated LZO compression patch
Stephen White
swhite at decisionsoft.com
Tue Mar 11 10:42:18 GMT 2003
For those interested in LZO compression I've attached an updated copy of
my patch, that should apply to the current CVS. I haven't really tested
this version beyond 'make check' yet, so no guarantees, but I think it's
right.
export DISTCC_COMPRESS=1 before running the client to enable
compression.
Don't forget to rerun ./autogen.sh after applying the patch.
--
Stephen White +44-1865-203192
DecisionSoft Limited http://www.decisionsoft.com
XML Development and Services
-------------- next part --------------
diff -N -aur --exclude=CVS distcc.clean/Makefile.in distcc.my.clean/Makefile.in
--- distcc.clean/Makefile.in Sun Mar 9 23:58:40 2003
+++ distcc.my.clean/Makefile.in Mon Mar 10 16:42:39 2003
@@ -78,6 +78,7 @@
$(MEN) \
$(pkgdoc_DOCS) \
$(popt_SRC) $(popt_HEADERS) \
+ $(lzo_SRC) $(lzo_HEADERS) \
$(SRC) $(HEADERS) \
$(test_SOURCE) \
$(bench_PY)
@@ -143,9 +144,9 @@
distnews = $(PACKAGE_TARNAME)-$(VERSION).NEWS
common_obj = src/trace.o src/util.o src/io.o src/exec.o src/arg.o \
-src/rpc.o src/tempfile.o src/bulk.o src/help.o src/filename.o \
-src/lock.o src/where.o src/hosts.o src/sendfile.o \
- src/snprintf.o src/timeval.o
+ src/rpc.o src/tempfile.o src/bulk.o src/bulk_lzo.o src/help.o \
+ src/filename.o src/lock.o src/where.o src/hosts.o \
+ src/sendfile.o src/snprintf.o src/timeval.o @BUILD_LZO@
distcc_obj = src/clinet.o src/clirpc.o src/cpp.o src/distcc.o \
src/implicit.o src/strip.o $(common_obj)
@@ -163,9 +164,9 @@
h_strip_obj = src/h_strip.o src/trace.o src/util.o src/arg.o src/filename.o src/strip.o
h_parsemask_obj = src/h_parsemask.o $(common_obj) src/access.o
-SRC = src/arg.c src/bulk.c src/clinet.c src/clirpc.c src/cpp.c \
- src/daemon.c src/distcc.c src/sendfile.c src/ssh.c \
- src/access.c \
+SRC = src/arg.c src/bulk.c src/clinet.c src/cpp.c, src/bulk_lzo.c \
+ src/daemon.c src/distcc.c src/sendfile.c src/access.c \
+ src/clirpc.c src/ssh.c \
src/dopt.c src/dparent.c src/exec.c src/filename.c \
src/h_argvtostr.c \
src/h_exten.c src/h_hosts.c src/h_issource.c src/h_parsemask.c \
@@ -179,14 +180,19 @@
HEADERS = src/access.h src/bulk.h src/clinet.h src/clirpc.h src/cpp.h \
src/distcc.h src/exitcode.h src/filename.h src/hosts.h \
src/io.h src/tempfile.h src/setuid.h \
- src/strip.h src/implicit.h \
- src/opt.h src/rpc.h \
- src/snprintf.h src/trace.h src/util.h \
- src/timeval.h \
+ src/strip.h src/implicit.h src/bulk_lzo.h \
+ src/opt.h src/rpc.h src/trace.h src/util.h src/bulk.h \
+ src/snprintf.h src/timeval.h \
src/exec.h src/lock.h src/where.h src/srvnet.h
MEN = man/distcc.1 man/distccd.1
+lzo_OBJS=lzo/minilzo.o
+
+lzo_HEADERS=lzo/minilzo.h lzo/lzoconf.h
+
+lzo_SRC=lzo/minilzo.c lzo/testmini.c
+
popt_OBJS=popt/findme.o popt/popt.o popt/poptconfig.o \
popt/popthelp.o popt/poptparse.o
@@ -272,7 +278,6 @@
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(h_strip_obj) $(LIBS)
-
## DIST targets
# The sub-targets copy (and if necessary, build) various files that
@@ -371,6 +376,7 @@
clean: clean-autoconf
rm -f src/*.o popt/*.o
rm -f $(check_PROGRAMS) $(bin_PROGRAMS)
+ make -C lzo clean
rm -rf testtmp
clean-autoconf:
diff -N -aur --exclude=CVS distcc.clean/configure.ac distcc.my.clean/configure.ac
--- distcc.clean/configure.ac Mon Mar 10 04:10:43 2003
+++ distcc.my.clean/configure.ac Mon Mar 10 16:08:59 2003
@@ -48,7 +48,11 @@
### Checks for configure options
AC_ARG_WITH(included-popt,
- AC_HELP_STRING([--with-included-popt], [use bundled popt library, not from system]))
+ [ --with-included-popt use bundled popt library, not from system])
+
+AC_ARG_WITH(included-lzo,
+ [ --with-included-lzo use bundled lzo library, not shared system library])
+# AC_HELP_STRING([--with-included-popt], [use bundled popt library, not from system]))
dnl Checks for programs
AC_PROG_CC
@@ -124,6 +128,7 @@
########################################################################
### Checks for libraries.
+
AC_CHECK_LIB(nsl, gethostbyname)
AC_CHECK_LIB(socket, connect)
AC_CHECK_LIB(resolv, hstrerror, , , [-lnsl -lsocket])
@@ -147,10 +152,24 @@
AC_MSG_RESULT(no)
fi
+if test x"$with_included_lzo" != x"yes" && test x"$with_included_lzo" != xno
+then
+ # If not explicitly requested, guess
+ AC_CHECK_LIB(lzo, lzo1x_1_compress, , [with_included_lzo=yes])
+ AC_CHECK_LIB(lzo, lzo1x_decompress_safe, , [with_included_lzo=yes])
+fi
+AC_CACHE_SAVE
+AC_MSG_CHECKING([whether to use included minilzo])
+if test x"$with_included_lzo" = x"yes"
+then
+ AC_MSG_RESULT($srcdir/lzo)
+ BUILD_LZO='$(lzo_OBJS)'
+ CPPFLAGS="$CPPFLAGS -I$srcdir/lzo"
+else
+ AC_MSG_RESULT(no)
+fi
-########################################################################
-# Checks for library functions, using libraries discovered above
CPPFLAGS="$CPPFLAGS -I$srcdir/src"
AC_CHECK_FUNCS([sendfile setsid flock lockf hstrerror strerror setuid setreuid])
@@ -223,6 +242,7 @@
AC_SUBST(LDFLAGS)
AC_SUBST(CPPFLAGS)
AC_SUBST(BUILD_POPT)
+AC_SUBST(BUILD_LZO)
dnl AC_DEFINE_UNQUOTED(PACKAGE, $PACKAGE, [Package name])
dnl AC_DEFINE_UNQUOTED(VERSION, $VERSION, [Package version])
AC_DEFINE_UNQUOTED(GNU_HOST, ["$host"], [Your gnu-style host triple])
diff -N -aur --exclude=CVS distcc.clean/lzo/README.LZO distcc.my.clean/lzo/README.LZO
--- distcc.clean/lzo/README.LZO Mon Dec 16 03:43:42 2002
+++ distcc.my.clean/lzo/README.LZO Fri Jan 24 17:05:34 2003
@@ -1,3 +1,12 @@
+This is a copy of the miniLZO distribution, it is included with distcc to ease compilation on systems that do not have the full lzo library installed. On systems with an lzo library available this will not be used.
+
+The only modification to this distribution is to rename 'minilzo.h' to 'lzo1x.h' so that the code can use the same '#include' directives regardless of which distribution is in use.
+
+The remainder of this file is the standard miniLZO readme.
+
+-- Stephen White (2003/1/24)
+
+
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
diff -N -aur --exclude=CVS distcc.clean/lzo/lzo1x.h distcc.my.clean/lzo/lzo1x.h
--- distcc.clean/lzo/lzo1x.h Thu Jan 1 01:00:00 1970
+++ distcc.my.clean/lzo/lzo1x.h Mon Dec 16 03:43:42 2002
@@ -0,0 +1,100 @@
+/* minilzo.h -- mini subset of the LZO real-time data compression library
+
+ This file is part of the LZO real-time data compression library.
+
+ Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer
+ Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer
+ Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer
+ Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer
+ Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer
+ Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer
+ Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer
+ All Rights Reserved.
+
+ The LZO library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU General Public License as
+ published by the Free Software Foundation; either version 2 of
+ the License, or (at your option) any later version.
+
+ The LZO library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with the LZO library; see the file COPYING.
+ If not, write to the Free Software Foundation, Inc.,
+ 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+ Markus F.X.J. Oberhumer
+ <markus at oberhumer.com>
+ http://www.oberhumer.com/opensource/lzo/
+ */
+
+/*
+ * NOTE:
+ * the full LZO package can be found at
+ * http://www.oberhumer.com/opensource/lzo/
+ */
+
+
+#ifndef __MINILZO_H
+#define __MINILZO_H
+
+#define MINILZO_VERSION 0x1080
+
+#ifdef __LZOCONF_H
+# error "you cannot use both LZO and miniLZO"
+#endif
+
+#undef LZO_HAVE_CONFIG_H
+#include "lzoconf.h"
+
+#if !defined(LZO_VERSION) || (LZO_VERSION != MINILZO_VERSION)
+# error "version mismatch in header files"
+#endif
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/***********************************************************************
+//
+************************************************************************/
+
+/* Memory required for the wrkmem parameter.
+ * When the required size is 0, you can also pass a NULL pointer.
+ */
+
+#define LZO1X_MEM_COMPRESS LZO1X_1_MEM_COMPRESS
+#define LZO1X_1_MEM_COMPRESS ((lzo_uint32) (16384L * lzo_sizeof_dict_t))
+#define LZO1X_MEM_DECOMPRESS (0)
+
+
+/* compression */
+LZO_EXTERN(int)
+lzo1x_1_compress ( const lzo_byte *src, lzo_uint src_len,
+ lzo_byte *dst, lzo_uintp dst_len,
+ lzo_voidp wrkmem );
+
+/* decompression */
+LZO_EXTERN(int)
+lzo1x_decompress ( const lzo_byte *src, lzo_uint src_len,
+ lzo_byte *dst, lzo_uintp dst_len,
+ lzo_voidp wrkmem /* NOT USED */ );
+
+/* safe decompression with overrun testing */
+LZO_EXTERN(int)
+lzo1x_decompress_safe ( const lzo_byte *src, lzo_uint src_len,
+ lzo_byte *dst, lzo_uintp dst_len,
+ lzo_voidp wrkmem /* NOT USED */ );
+
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif /* already included */
+
diff -N -aur --exclude=CVS distcc.clean/src/bulk.c distcc.my.clean/src/bulk.c
--- distcc.clean/src/bulk.c Wed Feb 5 12:05:35 2003
+++ distcc.my.clean/src/bulk.c Tue Mar 11 09:37:25 2003
@@ -35,7 +35,6 @@
* keep writing), but we can't send from a fifo, because we wouldn't
* know how many bytes were coming.
*
- * @todo Optionally support lzo compression
**/
#include "config.h"
@@ -56,6 +55,7 @@
#include "io.h"
#include "rpc.h"
#include "bulk.h"
+#include "bulk_lzo.h"
#include "time.h"
#include "timeval.h"
@@ -64,7 +64,6 @@
# define O_BINARY 0
#endif
-
/**
* Open a file for read, and also put its size into @p fsize.
*
@@ -133,7 +132,7 @@
* Wrapper around dcc_x_file().
**/
int dcc_x_file_timed(int ofd, const char *fname, const char *token,
- size_t *size_out)
+ size_t *size_out, int use_lzo)
{
struct timeval before, after;
int ret;
@@ -142,7 +141,7 @@
if (gettimeofday(&before, NULL))
rs_log_warning("gettimeofday failed");
- ret = dcc_x_file(ofd, fname, token, &size);
+ ret = dcc_x_file(ofd, fname, token, &size, use_lzo);
if (size_out)
*size_out = size; /* separate to allow for size_out NULL */
@@ -171,7 +170,7 @@
* transmitted.
**/
int dcc_x_file(int ofd, const char *fname, const char *token,
- size_t *size_out)
+ size_t *size_out, int use_lzo)
{
int ifd;
off_t f_size;
@@ -189,6 +188,7 @@
goto failed;
if (f_size) {
+ if (!use_lzo) {
#ifdef HAVE_SENDFILE
if (dcc_pump_sendfile(ofd, ifd, (size_t) f_size))
goto failed;
@@ -196,6 +196,10 @@
if (dcc_pump_readwrite(ofd, ifd, (size_t) f_size))
goto failed;
#endif
+ } else {
+ if (dcc_x_file_body_pack(ofd,ifd,(size_t) f_size))
+ goto failed;
+ }
}
if (ifd != -1)
@@ -217,7 +221,7 @@
*
* @param filename local filename to create.
**/
-int dcc_r_file(int ifd, const char *filename, size_t len)
+int dcc_r_file(int ifd, const char *filename, size_t len, int use_lzo)
{
int ofd;
int ret, close_ret;
@@ -233,7 +237,7 @@
return -1;
}
- ret = dcc_r_file_body(ofd, ifd, len);
+ ret = dcc_r_file_body(ofd, ifd, len, use_lzo);
close_ret = dcc_close(ofd);
if (!ret && !close_ret) {
@@ -257,18 +261,21 @@
* The output file descriptor is not closed, because this routine is
* also used for copying onto stderr.
**/
-int dcc_r_file_body(int ofd, int ifd, size_t len)
+int dcc_r_file_body(int ofd, int ifd, size_t len, int use_lzo)
{
- return dcc_pump_readwrite(ofd, ifd, (size_t) len);
+ if (use_lzo) {
+ return dcc_r_file_body_unpack(ofd,ifd,(size_t) len);
+ } else {
+ return dcc_pump_readwrite(ofd, ifd, (size_t) len);
+ }
}
-
/**
* Receive a file and print timing statistics. Only used for big files.
*
* Wrapper around dcc_r_file().
**/
-int dcc_r_file_timed(int ifd, const char *fname, size_t size)
+int dcc_r_file_timed(int ifd, const char *fname, size_t size, int use_lzo)
{
struct timeval before, after;
int ret;
@@ -276,7 +283,7 @@
if (gettimeofday(&before, NULL))
rs_log_warning("gettimeofday failed");
- ret = dcc_r_file(ifd, fname, size);
+ ret = dcc_r_file(ifd, fname, size, use_lzo);
if (gettimeofday(&after, NULL)) {
rs_log_warning("gettimeofday failed");
diff -N -aur --exclude=CVS distcc.clean/src/bulk.h distcc.my.clean/src/bulk.h
--- distcc.clean/src/bulk.h Tue Feb 4 11:58:53 2003
+++ distcc.my.clean/src/bulk.h Tue Mar 11 09:33:08 2003
@@ -20,11 +20,9 @@
* USA
*/
-int dcc_r_file(int ifd, const char *filename, size_t);
-int dcc_r_file_body(int ofd, int ifd, size_t len);
-int dcc_r_fifo(int ifd, const char *fifo_name, size_t len);
-int dcc_x_file(int ofd, const char *fname, const char *token, size_t *);
+int dcc_r_file(int ifd, const char *filename, size_t, int use_lzo);
+int dcc_r_file_body(int ofd, int ifd, size_t len, int use_lzo);
+int dcc_x_file(int ofd, const char *fname, const char *token, size_t *, int use_lzo);
int dcc_x_file_timed(int ofd, const char *fname, const char *token,
- size_t *size_out);
-
-int dcc_r_file_timed(int ifd, const char *fname, size_t size);
+ size_t *size_out, int use_lzo);
+int dcc_r_file_timed(int ifd, const char *fname, size_t size, int use_lzo);
diff -N -aur --exclude=CVS distcc.clean/src/bulk_lzo.c distcc.my.clean/src/bulk_lzo.c
--- distcc.clean/src/bulk_lzo.c Thu Jan 1 01:00:00 1970
+++ distcc.my.clean/src/bulk_lzo.c Tue Mar 11 09:37:47 2003
@@ -0,0 +1,249 @@
+/* -*- c-file-style: "java"; indent-tabs-mode: nil -*-
+ *
+ * distcc -- A simple distributed compiler system
+ * $Header: /cvsroot/distcc/src/bulk.c,v 1.23 2002/11/01 07:58:31 mbp Exp $
+ *
+ * Copyright (C) 2002 by Stephen White <swhite-distcc at decisionsoft.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA
+ */
+
+/**
+ * @file
+ *
+ * Bulk file transfer, used for sending .i, .o files etc.
+ *
+ * These are the versions of the routines that carry out LZO compression,
+ * see bulk.c for more information.
+ *
+ * <swhite-distcc at decisionsoft.com>
+ **/
+
+#include "config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <assert.h>
+#include <string.h>
+#include <fcntl.h>
+#include <errno.h>
+
+#include <sys/stat.h>
+
+#include "distcc.h"
+#include "trace.h"
+#include "io.h"
+#include "rpc.h"
+
+#include <lzoconf.h>
+#include <lzo1x.h>
+
+#include "bulk_lzo.h"
+
+/**
+ * Memory for use by compression/decompression
+ */
+#define HEAP_ALLOC(var,size) \
+ lzo_align_t __LZO_MMODEL var [ ((size) + (sizeof(lzo_align_t) - 1)) / sizeof(lzo_align_t) ]
+
+#define LZO1X_1_MEM_COMPRESS ((lzo_uint32) (16384L * lzo_sizeof_dict_t))
+
+static HEAP_ALLOC(wrkmem,LZO1X_1_MEM_COMPRESS);
+
+#define min(x,y) (x>y?y:x)
+#define io_len(x) (min(SSIZE_MAX,x))
+
+/**
+ * Fills a buffer with up to 'size' bytes from fd.
+ * returns the number of bytes read
+ */
+size_t buf_from_fd(char *buf, int fd, size_t size) {
+ size_t n = 0;
+
+ while (n < size) {
+ int bytes = read(fd,buf + n, io_len(size) - n);
+
+ if (bytes <= 0) {
+ break;
+ } else {
+ n+=bytes;
+ }
+ }
+
+ return n;
+}
+
+/**
+ * Sends data from a buffer to a file descriptor
+ * returns the number of bytes sent.
+ */
+size_t buf_to_fd(char *buf, int fd, size_t size) {
+ size_t n = 0;
+ while (n < size) {
+ int bytes = write(fd,buf + n, io_len(size - n));
+
+ if (bytes < 0) {
+ break;
+ } else {
+ n+=bytes;
+ }
+ }
+
+ return n;
+}
+
+/**
+ * Like dcc_r_file_body but compress with lzo
+ */
+int dcc_x_file_body_pack(int ofd, int ifd, size_t len)
+{
+ size_t n = min(len,DISTCC_COMPRESS_BLOCK_SIZE);
+ size_t out_len = n + n / 64 + 16 + 3;
+ char *in_buf = malloc(n);
+ char *out_buf = malloc(out_len);
+
+ if ((in_buf == NULL) || (out_buf == NULL)) {
+ if (in_buf != NULL) { free(in_buf); }
+ if (out_buf != NULL) { free(out_buf); }
+ rs_log_warning("Memory allocation failed");
+ return -1;
+ }
+
+ while (len > 0) {
+ /* Read maximum of DISTCC_COMPRESS_BLOCK_SIZE from file descriptor
+ * into in_buf */
+ n = buf_from_fd(in_buf,ifd,min(len,DISTCC_COMPRESS_BLOCK_SIZE));
+
+ if (n <= 0) {
+ rs_log_warning("error reading from file");
+ break;
+ }
+
+ /* We've read something from the file, compress it, send it to
+ * the recipient and then repeat. */
+ len -= n;
+
+ /* Compress */
+ if (lzo1x_1_compress(in_buf,n,out_buf,&out_len,wrkmem) != LZO_E_OK) {
+ /* Internal error - shouldn't happen! */
+ rs_log_warning("LZO compress internal error");
+ }
+
+ /* Tell the recipient how long this compressed block is,
+ * and how much data it should unpack to */
+ if (dcc_x_token_int(ofd, "LZOL", out_len) ||
+ dcc_x_token_int(ofd, "OUTL", n)) {
+ rs_log_warning("Failed to send token");
+ free(in_buf);
+ free(out_buf);
+ return -1;
+ }
+
+ /* Write from out_buf to output stream */
+ if (buf_to_fd(out_buf,ofd,out_len) < out_len) {
+ rs_log_warning("failed to send file");
+ break;
+ }
+ }
+
+ free(in_buf);
+ free(out_buf);
+
+ if (len > 0) {
+ rs_log_warning("failed to send file (%i bytes unsent)",len);
+ return -1;
+ }
+
+ /* send LZOL block of length 0 .. to indicate EOF */
+ if (dcc_x_token_int(ofd, "LZOL", 0)) {
+ return -1;
+ }
+
+ rs_trace("LZO - done pack and send (%i packed bytes from %i bytes)",out_len,len);
+
+ return 0;
+}
+
+/**
+ * Like dcc_r_file_body but uncompress with lzo
+ */
+int dcc_r_file_body_unpack(int ofd, int ifd, size_t len)
+{
+ size_t in_len, n;
+ int r;
+
+ /* Read compressed LZO blocks until we reach one of length 0 ..
+ * which marks the EOF */
+ while (!dcc_r_token_int(ifd, "LZOL", &in_len) && in_len) {
+ char *out_buf = NULL, *in_buf = NULL;
+ size_t out_len;
+
+ if (dcc_r_token_int(ifd, "OUTL", &out_len)) {
+ rs_log_warning("Missing OUTL token");
+ return -1;
+ }
+
+ out_buf = malloc(out_len);
+ in_buf = malloc(in_len);
+
+ if ((in_buf == NULL) || (out_buf == NULL)) {
+ if (in_buf != NULL) { free(in_buf); }
+ if (out_buf != NULL) { free(out_buf); }
+ rs_log_warning("Memory allocation failed");
+ return -1;
+ }
+
+ rs_trace("reading packed stream (len %i)",in_len);
+
+ /* Read in_len bytes from file descriptor into in_buf */
+ if (buf_from_fd(in_buf,ifd,in_len) < in_len) {
+ rs_log_warning("error during read");
+ free(in_buf);
+ free(out_buf);
+ break;
+ }
+
+ /* decompress */
+ r = lzo1x_decompress_safe(in_buf,in_len,out_buf,&out_len,wrkmem);
+ if (r != LZO_E_OK) {
+ /* Internal error - shouldn't happen! */
+ rs_log_warning("LZO decompress internal error %i",r);
+ }
+
+ free(in_buf);
+
+ /* Write from out_buf to output stream */
+ n = buf_to_fd(out_buf,ofd,out_len);
+
+ if (n < out_len) {
+ rs_log_warning("error during write");
+ }
+
+ len-=n;
+
+ free(out_buf);
+ }
+
+ rs_trace("LZO decompress done");
+
+ if (len != 0) {
+ rs_log_warning("unpack fell short by %i bytes",len);
+ return -1;
+ }
+
+ return 0;
+}
diff -N -aur --exclude=CVS distcc.clean/src/bulk_lzo.h distcc.my.clean/src/bulk_lzo.h
--- distcc.clean/src/bulk_lzo.h Thu Jan 1 01:00:00 1970
+++ distcc.my.clean/src/bulk_lzo.h Tue Mar 11 09:37:34 2003
@@ -0,0 +1,29 @@
+/* -*- c-file-style: "java"; indent-tabs-mode: nil -*-
+ *
+ * distcc -- A simple distributed compiler system
+ * $Header: /cvsroot/distcc/src/bulk.h,v 1.1 2002/08/31 22:32:20 mbp Exp $
+ *
+ * Copyright (C) 2002 by Martin Pool <mbp at samba.org>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA
+ */
+
+/* When sending a file, compress in blocks of 64K. Changing this value as you
+ *see fit, the recieving process does not make any assumptions based on it. */
+#define DISTCC_COMPRESS_BLOCK_SIZE 65536
+
+int dcc_x_file_body_pack(int ofd, int ifd, size_t len);
+int dcc_r_file_body_unpack(int ofd, int ifd, size_t len);
diff -N -aur --exclude=CVS distcc.clean/src/clirpc.c distcc.my.clean/src/clirpc.c
--- distcc.clean/src/clirpc.c Tue Feb 25 11:08:07 2003
+++ distcc.my.clean/src/clirpc.c Mon Mar 10 16:31:37 2003
@@ -53,9 +53,9 @@
/*
* Transmit header for whole request.
*/
-static int dcc_x_req_header(int fd)
+static int dcc_x_req_header(int fd, int flags)
{
- return dcc_x_token_int(fd, "DIST", PROTO_VER);
+ return dcc_x_token_int(fd, "DIST", (flags << 16) | PROTO_VER);
}
@@ -85,12 +85,14 @@
char **argv,
pid_t cpp_pid,
int *status,
- const char *cpp_fname)
+ const char *cpp_fname,
+ int use_lzo)
{
long stime_usec, utime_usec;
int ret;
+ int flags = (use_lzo?FLAG_USE_LZO:0);
- if ((ret = dcc_x_req_header(net_fd))
+ if ((ret = dcc_x_req_header(net_fd,flags))
|| (ret = dcc_x_argv(net_fd, argv)))
return ret;
@@ -118,7 +120,7 @@
return EXIT_IO_ERROR;
}
- if ((ret = dcc_x_file_timed(net_fd, cpp_fname, "DOTI", NULL)))
+ if ((ret = dcc_x_file_timed(net_fd, cpp_fname, "DOTI", NULL, use_lzo)))
return ret;
rs_trace("client finished sending request to server");
@@ -144,13 +146,14 @@
char **argv,
pid_t cpp_pid,
int *status,
- const char *cpp_fname)
+ const char *cpp_fname,
+ int use_lzo)
{
int ret;
tcp_cork_sock(net_fd, 1);
- ret = dcc_send_job(net_fd, argv, cpp_pid, status, cpp_fname);
+ ret = dcc_send_job(net_fd, argv, cpp_pid, status, cpp_fname, use_lzo);
tcp_cork_sock(net_fd, 0);
@@ -158,7 +161,7 @@
}
-int dcc_retrieve_results(int net_fd, int *status, const char *output_fname)
+int dcc_retrieve_results(int net_fd, int *status, const char *output_fname, int use_lzo)
{
int len;
int ret;
@@ -167,9 +170,9 @@
if ((ret = dcc_r_result_header(net_fd))
|| (ret = dcc_r_cc_status(net_fd, status))
|| (ret = dcc_r_token_int(net_fd, "SERR", &len))
- || (ret = dcc_r_file_body(STDERR_FILENO, net_fd, len))
+ || (ret = dcc_r_file_body(STDERR_FILENO, net_fd, len, use_lzo))
|| (ret = dcc_r_token_int(net_fd, "SOUT", &len))
- || (ret = dcc_r_file_body(STDOUT_FILENO, net_fd, len))
+ || (ret = dcc_r_file_body(STDOUT_FILENO, net_fd, len, use_lzo))
|| (ret = dcc_r_token_int(net_fd, "DOTO", &o_len)))
return ret;
@@ -179,7 +182,7 @@
* sends an 0 byte file on failure anyhow. */
if (o_len)
- return dcc_r_file_timed(net_fd, output_fname, o_len);
+ return dcc_r_file_timed(net_fd, output_fname, o_len, use_lzo);
else {
rs_log_notice("skipping retrieval of 0 byte object file %s",
output_fname);
diff -N -aur --exclude=CVS distcc.clean/src/clirpc.h distcc.my.clean/src/clirpc.h
--- distcc.clean/src/clirpc.h Tue Feb 25 11:08:07 2003
+++ distcc.my.clean/src/clirpc.h Mon Mar 10 16:35:26 2003
@@ -2,6 +2,7 @@
char **argv,
pid_t cpp_pid,
int *status,
- const char *cpp_fname);
+ const char *cpp_fname,
+ int use_lzo);
-int dcc_retrieve_results(int net_fd, int *status, const char *output_fname);
+int dcc_retrieve_results(int net_fd, int *status, const char *output_fname, int use_lzo);
diff -N -aur --exclude=CVS distcc.clean/src/daemon.c distcc.my.clean/src/daemon.c
--- distcc.clean/src/daemon.c Mon Jan 27 14:02:00 2003
+++ distcc.my.clean/src/daemon.c Mon Jan 27 15:08:56 2003
@@ -60,7 +60,7 @@
#include "opt.h"
#include "setuid.h"
#include "srvnet.h"
-
+#include "lzoconf.h"
/* for trace.c */
char const *rs_program_name = "distccd";
@@ -93,6 +93,12 @@
* the right ownership. */
dccd_setup_log();
+ if (lzo_init() != LZO_E_OK)
+ {
+ printf("lzo_init() failed\n");
+ return EXIT_DISTCC_FAILED;
+ }
+
/* This test might need to be a bit more complex when we want to
* support ssh and stuff like that. Perhaps it would be better to
* always serve stdin, and then complain if it's a terminal? */
diff -N -aur --exclude=CVS distcc.clean/src/distcc.c distcc.my.clean/src/distcc.c
--- distcc.clean/src/distcc.c Wed Mar 5 02:02:22 2003
+++ distcc.my.clean/src/distcc.c Mon Mar 10 16:33:32 2003
@@ -59,9 +59,9 @@
#include "where.h"
#include "lock.h"
#include "cpp.h"
+#include "lzoconf.h"
#include "clirpc.h"
-
/* Name of this program, for trace.c */
const char *rs_program_name = "distcc";
@@ -125,8 +125,6 @@
DISTCC_DEFAULT_PORT);
}
-
-
/**
* Pass a compilation across the network.
*
@@ -167,6 +165,13 @@
{
int net_fd;
int ret;
+ int use_lzo = (dcc_getenv_bool("DISTCC_COMPRESS", 0));
+
+ if (use_lzo && (lzo_init() != LZO_E_OK))
+ {
+ printf("lzo_init() failed\n");
+ return 3;
+ }
*status = 0;
@@ -177,12 +182,12 @@
/* This waits for cpp and puts its status in *status. If cpp failed, then
* the connection will have been dropped and we need not bother trying to
* get any response from the server. */
- ret = dcc_send_job_corked(net_fd, argv, cpp_pid, status, cpp_fname);
+ ret = dcc_send_job_corked(net_fd, argv, cpp_pid, status, cpp_fname, use_lzo);
dcc_unlock(xmit_lock_fd);
-
+
if (ret == 0 && *status == 0) {
- ret = dcc_retrieve_results(net_fd, status, output_fname);
+ ret = dcc_retrieve_results(net_fd, status, output_fname, use_lzo);
}
/* Close socket so that the server can terminate, rather than
diff -N -aur --exclude=CVS distcc.clean/src/distcc.h distcc.my.clean/src/distcc.h
--- distcc.clean/src/distcc.h Sat Feb 22 01:41:31 2003
+++ distcc.my.clean/src/distcc.h Mon Mar 10 16:05:39 2003
@@ -22,6 +22,8 @@
#define PROTO_VER 1
+#define FLAG_USE_LZO 0x1
+
#ifdef NORETURN
/* nothing */
#elif defined(__GNUC__)
diff -N -aur --exclude=CVS distcc.clean/src/hosts.c distcc.my.clean/src/hosts.c
--- distcc.clean/src/hosts.c Tue Feb 25 00:27:25 2003
+++ distcc.my.clean/src/hosts.c Mon Mar 10 16:05:39 2003
@@ -123,7 +123,6 @@
rs_log_warning("$DISTCC_HOSTS is not defined; can't distribute work");
return EXIT_BAD_HOSTSPEC;
}
-
return dcc_parse_hosts(where, ret_list, ret_nhosts);
}
diff -N -aur --exclude=CVS distcc.clean/src/serve.c distcc.my.clean/src/serve.c
--- distcc.clean/src/serve.c Wed Mar 5 02:02:22 2003
+++ distcc.my.clean/src/serve.c Mon Mar 10 16:34:24 2003
@@ -83,8 +83,10 @@
#include "srvnet.h"
#include "filename.h"
-
-static int dcc_r_request_header(int ifd)
+/**
+ * Check the DIST protocol version header and read any flags.
+ */
+static int dcc_r_request_header(int ifd, int *flags)
{
int vers;
@@ -93,7 +95,9 @@
return -1;
}
- if (vers != PROTO_VER) {
+ *flags = vers >> 16;
+
+ if ((vers & 0xffff) != PROTO_VER) {
rs_log_error("client version is %d, i am %d", vers, 1);
return -1;
}
@@ -222,7 +226,7 @@
**/
static int dcc_server_compile(char **argv, int netfd,
const char *temp_i,
- const char *temp_out, const char *temp_err)
+ const char *temp_out, const char *temp_err, int flags)
{
int i_size;
int ret;
@@ -236,7 +240,7 @@
rs_log_info("input file is %d bytes", i_size);
/* read input before spawning */
- if ((ret = dcc_r_file(netfd, temp_i, i_size)) != 0)
+ if ((ret = dcc_r_file(netfd, temp_i, i_size, flags & FLAG_USE_LZO)) != 0)
return ret;
return dcc_spawn_child(argv, &pid, "/dev/null", temp_out, temp_err);
@@ -288,6 +292,7 @@
const char *input_exten;
long u_us, s_us;
int i_size;
+ int flags;
pid_t cc_pid;
err_fname = dcc_make_tmpnam("cc", ".err");
@@ -320,7 +325,7 @@
/* Allow output to accumulate into big packets. */
tcp_cork_sock(netfd, 1);
- if ((ret = dcc_r_request_header(netfd))
+ if ((ret = dcc_r_request_header(netfd,&flags))
|| (ret = dcc_r_argv(netfd, &argv))
|| (ret = dcc_scan_args(argv, &orig_input, &orig_output, &argv)))
return ret;
@@ -340,7 +345,7 @@
temp_o = dcc_make_tmpnam("server", ".out");
if ((ret = dcc_r_token_int(netfd, "DOTI", &i_size))
- || (ret = dcc_r_file_timed(netfd, temp_i, (size_t) i_size))
+ || (ret = dcc_r_file_timed(netfd, temp_i, (size_t) i_size, flags & FLAG_USE_LZO))
|| (ret = dcc_set_input(argv, temp_i))
|| (ret = dcc_set_output(argv, temp_o)))
return ret;
@@ -353,13 +358,13 @@
if ((ret = dcc_x_result_header(netfd))
|| (ret = dcc_x_cc_status(netfd, status))
- || (ret = dcc_x_file(netfd, err_fname, "SERR", NULL))
- || (ret = dcc_x_file(netfd, out_fname, "SOUT", NULL))
+ || (ret = dcc_x_file(netfd, err_fname, "SERR", NULL, flags & FLAG_USE_LZO))
+ || (ret = dcc_x_file(netfd, out_fname, "SOUT", NULL, flags & FLAG_USE_LZO))
|| WIFSIGNALED(status)
|| WEXITSTATUS(status)) {
dcc_x_token_int(netfd, "DOTO", 0);
} else {
- ret = dcc_x_file_timed(netfd, temp_o, "DOTO", &o_size);
+ ret = dcc_x_file_timed(netfd, temp_o, "DOTO", &o_size, flags & FLAG_USE_LZO);
}
dcc_report_rusage(argv[0], u_us, s_us);
diff -N -aur --exclude=CVS distcc.clean/test/testdistcc.py distcc.my.clean/test/testdistcc.py
--- distcc.clean/test/testdistcc.py Thu Feb 27 01:20:37 2003
+++ distcc.my.clean/test/testdistcc.py Mon Mar 10 16:05:41 2003
@@ -861,7 +861,6 @@
def runTest(self):
msgs = self.runCmd("distcc nosuchcc -c testtmp.i", expectedResult=EXIT_COMPILER_MISSING)
self.assert_regexp(r'failed to exec.*No such file or directory', msgs)
-
class RemoteAssemble_Case(CompileHello_Case):
More information about the distcc
mailing list