access_table() challenge status

[Cole, Timothy D.]

> -----Original Message-----
> From:	Andrew Tridgell [SMTP:tridge at linuxcare.com]
> Sent:	Tuesday, January 18, 2000 7:01
> To:	Multiple recipients of list SAMBA-TECHNICAL
> Subject:	access_table() challenge status
> 
> Thanks for all the contributions!
> 
> The submission from Paul Mackerras is the leader so far. Mattys
> submission would be the best (by a long way!) if it handled the
> DENY_FCB case. Mattys certainly gives the most insight into how this
> stuff works.
> 
> Here is Mattys in case anyone missed it. It is very simple
> conceptually (only two ugly conditionals) but doesn't even attempt
> DENY_FCB.
> 
	Okay, I took the FCB code from my attempt after optimizing it a bit
more (I'm glad I factored it out when I first wrote it :P), and merged it
with Matty's.  The clarity of the code has suffered somewhat, but I think
the FCB stuff is going to be evil no matter how you handle it.  In any case,
here's the result:

	static int deny_mask(int deny) {
		switch (deny) {
		  case DENY_READ: return 1;
		  case DENY_WRITE: return 2;
		  case DENY_ALL: return 3;
		  default: return 0;
		}
	}

	static int mode_mask(int mode) {
		switch (mode) {
		  case DOS_OPEN_RDONLY:	return 1;
		  case DOS_OPEN_WRONLY: return 2;
		  case DOS_OPEN_RDWR: return 3;
		  default: return 0;
		}
	}

	static BOOL is_compatible(int old_deny, int old_mode, int new_deny,
	                        int new_mode, BOOL same_process, BOOL isexe)
	{
		if ( old_deny == DENY_DOS ) {
	        	return isexe || ( ( old_mode == DOS_OPEN_RDONLY )
			               && ( new_mode == DOS_OPEN_RDONLY ) )
			             || ( same_process && ( old_deny ==
DENY_DOS )
			               && ( new_deny == DENY_DOS ) );
		} else {
			return !( mode_mask(new_mode) & deny_mask(old_deny)
);
		}
	}

	static BOOL is_allowed(int new_deny, int new_mode, int old_deny,
	                        int old_mode, BOOL same_process, BOOL isexe)
	{
	        if ((old_deny == DENY_DOS) && (old_mode == DOS_OPEN_RDONLY)
	                && (new_mode != DOS_OPEN_RDONLY) && same_process &&
!isexe)
	                return 0;

	        return is_compatible(old_deny, old_mode, new_deny, new_mode,
	                                                same_process, isexe)
	                && is_compatible(new_deny, new_mode, old_deny,
old_mode,
	                                                same_process,
isexe);
	}


	static int access_table2(int new_deny,int old_deny,int old_mode,
				 BOOL same_pid, BOOL isexe)
	{
		BOOL r, w;

		if (isexe) {
			if ( old_deny == DENY_FCB ) switch (new_deny) {
			  case DENY_FCB:
			  case DENY_DOS:
				return same_pid ? AALL : AFAIL;
			  default:
				return AFAIL;
			} else if ( new_deny == DENY_FCB ) {
				return AFAIL;
			}
		} else {
			if ( old_deny == DENY_FCB ) switch (new_deny) {
			  case DENY_FCB:
			  case DENY_DOS:
			  case DENY_WRITE:
				if (same_pid) return ( new_deny ==
DENY_WRITE )
				                   ? AFAIL : AALL;
				return ( old_mode == DOS_OPEN_RDONLY ) ?
AREAD : AFAIL;
			  case DENY_NONE:
				if ( !same_pid && ( old_mode ==
DOS_OPEN_RDONLY ) ) {
					return AREAD;
				}
			  default:
				return AFAIL;
			} else if ( new_deny == DENY_FCB ) switch (old_deny)
{
			  case DENY_DOS:
				if (same_pid) {
					return ( old_mode != DOS_OPEN_RDONLY
)
					     ? AALL : AFAIL;
				}
				if ( deny_mask(new_deny) &
mode_mask(old_mode) ) {
					return AFAIL;
				}
			  case DENY_WRITE:
			  case DENY_NONE:
				return ( old_mode == DOS_OPEN_RDONLY ) ?
AREAD : AFAIL;
			  default:
				return AFAIL;
			}
		}

		r = is_allowed(new_deny, DOS_OPEN_RDONLY, old_deny,
old_mode, same_pid,
			       isexe);
		w = is_allowed(new_deny, DOS_OPEN_WRONLY, old_deny,
old_mode, same_pid,
			       isexe);
		
		if (r && w) return AALL;
		if (r) return AREAD;
		if (w) return AWRITE;
		return AFAIL;
	}