[Samba] Solaris 8, 64-bit compile problems with CVS SAMBA_2_2 as of 2002/02/05 16:13 EST

[Eric Boehm]

I've run into the following problems compiling the SAMBA_2_2 label
with Sun WorkShop 6, Update 2 as a 64-bit build

My configure line

CC=cc CFLAGS='-xtarget=ultra2 -xarch=v9a' ./configure 

1. 64-bit swat is still broken because the crypt function isn't
   prototyped correctly. I first report this in April 2001

   http://marc.theaimsgroup.com/?l=samba-technical&m=98708044014344&w=2  

   I had some private correspondence with Jerry Carter about this. THe
   workaround was to add CPPFLAGS="-D__EXTENSIONS__"

   I pursued this with Sun and eventually found it as a bug in the
   crypt man page. The man page says to include unistd.h. You should
   include crypt.h instead. /usr/include/crypt.h exists on Solaris
   2.6, Solaris 7 and Solaris 8. I don't have access to earlier
   versions of Solaris to check.

   This could probably be handled by configure.


2. 64-bit build fails on libnss_winbind

   Linking bin/winbindd
   Compiling nsswitch/winbind_nss.c with -KPIC
   "nsswitch/winbind_nss.c", line 855: warning: argument #4 is incompatible with prototype:
           prototype: pointer to int : "nsswitch/winbind_nss.c", line 589
           argument : pointer to unsigned long

[several errors deleted]

   Compiling nsswitch/wb_common.c with -KPIC
   Compiling nsswitch/winbind_nss_solaris.c with -KPIC
   Linking nsswitch/libnss_winbind.so
   ld: fatal: file nsswitch/winbind_nss.po: wrong ELF class: ELFCLASS64
   ld: fatal: File processing errors. No output written to nsswitch/libnss_winbind.
   so
   make: *** [nsswitch/libnss_winbind.so] Error 1

After changing the rules in the Makefile, I see

cc  -h nsswitch/libnss_winbind.so -G -o nsswitch/libnss_winbind.so nsswitch/winbind_nss.po nsswitch/wb_common.po nsswitch/winbind_nss_solaris.po
ld: fatal: file nsswitch/winbind_nss.po: wrong ELF class: ELFCLASS64
ld: fatal: File processing errors. No output written to nsswitch/libnss_winbind.so
make: *** [nsswitch/libnss_winbind.so] Error 1

The problem is that the CFLAGS='-xtarget=ultra2 -xarch=v9a' need to be
included in the command line. It seems like this should probably be
done for most shared libraries, or at least when SHLD=${CC}. This is
probably the *WRONG* thing to do for HP-UX, perhaps CFLAGS needs to be
included in LDSHFLAGS. I'll let other more familiar with the platforms
make the best decision.

Here's a potential patch

--- Makefile.in.orig Wed Feb  6 12:55:32 2002
+++ Makefile.in.     Wed Feb  6 12:55:13 2002
@@ -608,7 +608,7 @@
 
 bin/smbwrapper. at SHLIBEXT@: $(PICOBJS)
        @echo Linking shared library $@
-       @$(SHLD) @LDSHFLAGS@ -o $@ $(PICOBJS) $(LIBS)
+       @$(SHLD) @CFLAGS@ @LDSHFLAGS@ -o $@ $(PICOBJS) $(LIBS)
 
 bin/smbwrapper.32. at SHLIBEXT@: $(PICOBJS32)
        @echo Linking shared library $@
@@ -616,7 +616,7 @@
 
 libsmbclient: $(LIBSMBCLIENT_PICOBJS)
        @echo Linking libsmbclient shared library bin/$@. at SHLIBEXT@
-       @$(SHLD) @LDSHFLAGS@ -o  bin/$@. at SHLIBEXT@ \
+       @$(SHLD) @CFLAGS@ @LDSHFLAGS@ -o  bin/$@. at SHLIBEXT@ \
                $(LIBSMBCLIENT_PICOBJS) $(LIBS) \
                @SONAMEFLAG at libsmbclient.so.$(LIBSMBCLIENT_MAJOR)
        @echo Linking libsmbclient non-shared library bin/$@.a
@@ -628,11 +628,11 @@
 
 bin/pam_smbpass. at SHLIBEXT@: $(PAM_SMBPASS_OBJ)
        @echo Linking shared library $@
-       $(SHLD) @LDSHFLAGS@ -symbolic -o $@ $(PAM_SMBPASS_OBJ) -lpam $(LIBS) -lc
+       $(SHLD) @CFLAGS@ @LDSHFLAGS@ -symbolic -o $@ $(PAM_SMBPASS_OBJ) -lpam $(LIBS) -lc
 
 nsswitch/libnss_wins.so: $(NSS_OBJ)
        @echo "Linking $@"
-       @$(SHLD) @LDSHFLAGS@ -o $@ $(NSS_OBJ) -lc
+       @$(SHLD) @CFLAGS@ @LDSHFLAGS@ -o $@ $(NSS_OBJ) -lc
 
 bin/winbindd: $(WINBINDD_OBJ) bin/.dummy
        @echo Linking $@
@@ -640,11 +640,11 @@
 
 nsswitch/libnss_winbind.so: $(WINBIND_NSS_PICOBJS)
        @echo "Linking $@"
-       @$(SHLD) @LDSHFLAGS@ -o $@ $(WINBIND_NSS_PICOBJS)
+       @$(SHLD) @CFLAGS@ @LDSHFLAGS@ -o $@ $(WINBIND_NSS_PICOBJS)
 
 nsswitch/pam_winbind.so: $(PAM_WINBIND_OBJ) bin/.dummy
        @echo Linking $@
-       @$(SHLD) @LDSHFLAGS@ -o $@ $(PAM_WINBIND_OBJ)
+       @$(SHLD) @CFLAGS@ @LDSHFLAGS@ -o $@ $(PAM_WINBIND_OBJ)
 
 bin/wbinfo: $(WBINFO_OBJ) $(PARAM_OBJ) $(LIB_OBJ) $(NOPROTO_OBJ) $(UBIQX_OBJ) bin/.dummy
        @echo Linking $@


The prototype mismatches seemed be cause by function calls using
size_t as the type for buflen when the function prototypes specify
"int *buflen".

Here's a possible patch:
--- winbind_nss.c.orig  Wed Feb  6 13:02:04 2002
+++ winbind_nss.c       Wed Feb  6 13:23:57 2002
@@ -522,7 +522,7 @@
    are the pointers passed in by the C library to the _nss_ntdom_*
    functions. */
 
-static char *get_static(char **buffer, int *buflen, int len)
+static char *get_static(char **buffer, size_t *buflen, int len)
 {
        char *result;
 
@@ -588,7 +588,7 @@
 
 static NSS_STATUS fill_pwent(struct passwd *result,
                                  struct winbindd_pw *pw,
-                                 char **buffer, int *buflen)
+                                 char **buffer, size_t *buflen)
 {
        /* User name */
 
@@ -663,7 +663,7 @@
    Return NSS_STATUS_TRYAGAIN if we run out of memory. */
 
 static int fill_grent(struct group *result, struct winbindd_gr *gr,
-                     char *gr_mem, char **buffer, int *buflen)
+                     char *gr_mem, char **buffer, size_t *buflen)
 {
        fstring name;
        int i;

-- 
Eric M. Boehm                  /"\  ASCII Ribbon Campaign
boehm at nortelnetworks.com       \ /  No HTML or RTF in mail
                                X   No proprietary word-processing
                               / \  files in mail
                              /   \ Respect Open Standards