[ccache] Two patches.
Kaz Kylheku
kkylheku at gmail.com
Wed Dec 13 04:06:06 GMT 2006
These were produced against sources from the 2.4 tarball. They are
unrelated and can be applied in either order, modulo some hunk offset
warnings. Strip level -p2 is required.
depends-bugfix
This fixes a bug that occurs when the caller is using -MD or -MMD to
generate dependencies, and is relying on the default behavior of the
compiler with respect to the choice of target name in the dependency
file, or the name of the dependency file itself.
You /don't/ need this fix if you don't use -MD or -MMD, or if you
always use both -MT and -MF in conjunction with them.
The problem addressed is that when the compiler is called on to
preprocess with the -E option, and is consequently not given the
output file name with -o, its algorithm for choosing the default
dependency target name and dependency file name is altered: these
names, ordinarily derived from the full path of the output file, are
instead derived from the basename of the input file.
NOTE: this fix is gcc-specific in the sense that /if/ the compiler
command line contains -MD or -MMD, the semantics of these options are
assumed to be those of gcc, and ccache takes the liberty of adding -MT
or -MF options.
multiple-caches
This patch adds a new feature. Using CCACHE_EXTRA_DIRS you can specify
a colon-separated list of up to 32 pathnames which give the locations
of additional caches. The regular CCACHE_DIR is then referred to as
the primary cache.
When this is specified, ccache will first look for the object file in
your primary cache. If that lookup fails, the extra caches are
searched.
The extra caches are treated somewhat differently from the normal
cache. If the path name to the object file does not exist in them, it
is not created. Object files are never moved into these caches, and
statistics are not updated there either. A cache hit from an extra
cache is not counted as a hit in the statistics of your primary cache,
but an overall miss is counted against your primary one.
With this feature, developers can take advantage of "official build"
caches, while still locally caching material which differs from those
caches, something that is not possible using merely CCACHE_READONLY
against an official cache. Moreover, the behavior is truly read-only
w.r.t. these extra caches: no stats updates, no path creation. Users
don't have to have write permissions to the caches in
CCACHE_EXTRA_CACHES at all.
Comments, flames, fixes: kkylheku at gmail.com
I'm not subscribed to this mailing list, but I will check the archives.
-------------- next part --------------
Index: src/ccache-2.4/ccache.c
===================================================================
--- src.orig/ccache-2.4/ccache.c 2006-12-13 07:49:25.000000000 -0800
+++ src/ccache-2.4/ccache.c 2006-12-13 08:06:55.138558704 -0800
@@ -65,6 +65,15 @@
/* can we safely use the unification hashing backend? */
static int enable_unify;
+/* is gcc being asked to output dependencies? */
+static int generating_dependencies;
+
+/* is the dependency makefile name overridden with -MF? */
+static int dependency_filename_specified;
+
+/* is the dependency makefile target name specified with -MQ or -MF? */
+static int dependency_target_specified;
+
/* a list of supported file extensions, and the equivalent
extension for code that has been through the pre-processor
*/
@@ -702,6 +711,25 @@
continue;
}
+ /* These options require special handling, because they
+ behave differently with gcc -E, when the output
+ file is not specified. */
+
+ if (strcmp(argv[i], "-MD") == 0 || strcmp(argv[i], "-MMD") == 0) {
+ generating_dependencies = 1;
+ } else if (strcmp(argv[i], "-MF") == 0) {
+ dependency_filename_specified = 1;
+ } else if (strcmp(argv[i], "-MQ") == 0 || strcmp(argv[i], "-MT") == 0) {
+ dependency_target_specified = 1;
+ }
+
+ /* other options */
+ if (argv[i][0] == '-') {
+ args_add(stripped_args, argv[i]);
+ continue;
+ }
+
+
/* options that take an argument */
{
const char *opts[] = {"-I", "-include", "-imacros", "-iprefix",
@@ -730,12 +758,6 @@
if (opts[j]) continue;
}
- /* other options */
- if (argv[i][0] == '-') {
- args_add(stripped_args, argv[i]);
- continue;
- }
-
/* if an argument isn't a plain file then assume its
an option, not an input file. This allows us to
cope better with unusual compiler options */
@@ -814,6 +836,28 @@
p[2] = 0;
}
+ /* If dependencies are generated, configure the preprocessor */
+
+ if (generating_dependencies) {
+ if (!dependency_filename_specified) {
+ char *default_depfile_name = x_strdup(output_file);
+ char *p = strrchr(default_depfile_name, '.');
+ p[1] = 'd';
+ p[2] = 0;
+ args_add(stripped_args, "-MF");
+ args_add(stripped_args, default_depfile_name);
+ }
+
+ if (!dependency_target_specified) {
+ char *default_target_name = x_strdup(output_file);
+ char *p = strrchr(default_target_name, '.');
+ p[1] = 'd';
+ p[2] = 0;
+ args_add(stripped_args, "-MT");
+ args_add(stripped_args, default_target_name);
+ }
+ }
+
/* cope with -o /dev/null */
if (strcmp(output_file,"/dev/null") != 0 && stat(output_file, &st) == 0 && !S_ISREG(st.st_mode)) {
cc_log("Not a regular file %s\n", output_file);
-------------- next part --------------
Index: src/ccache-2.4/ccache.c
===================================================================
--- src.orig/ccache-2.4/ccache.c 2006-12-13 08:06:55.000000000 -0800
+++ src/ccache-2.4/ccache.c 2006-12-13 12:26:47.284192272 -0800
@@ -26,6 +26,9 @@
/* the base cache directory */
char *cache_dir = NULL;
+/* additional read-only cache directories */
+char *extra_dirs[MAX_DIRS] = { 0 };
+
/* the directory for temporary files */
static char *temp_dir = NULL;
@@ -44,9 +47,15 @@
/* the source file */
static char *input_file;
+/* output of the hashing function */
+static char *hash;
+
/* the name of the file containing the cached object code */
static char *hashname;
+/* name fo a file containing cached object code in one of the extra caches */
+static char *hashname_extra;
+
/* the extension of the file after pre-processing */
static const char *i_extension;
@@ -257,26 +266,63 @@
free(path_stderr);
}
+/* Compute a multi-level hash name, possibly creating the directories,
+ and possibly recording the result in global variables. */
+
+static void do_multilevel_name(char *cache_dir, int set_names, int create_dirs)
+{
+ int i;
+ int nlevels = 2;
+ char *s, *hash_dir;
+ char *sep = (*cache_dir == 0) ? "" : "/";
+
+ if ((s = getenv("CCACHE_NLEVELS"))) {
+ nlevels = atoi(s);
+ if (nlevels < 1) nlevels = 1;
+ if (nlevels > 8) nlevels = 8;
+ }
+
+ x_asprintf(&hash_dir, "%s%s%c", cache_dir, sep, hash[0]);
+
+ if (set_names)
+ x_asprintf(&stats_file, "%s/stats", hash_dir);
+
+ for (i=1; i<nlevels; i++) {
+ char *p;
+ if (create_dirs && create_dir(hash_dir) != 0) {
+ cc_log("failed to create %s\n", hash_dir);
+ failed();
+ }
+ x_asprintf(&p, "%s/%c", hash_dir, hash[i]);
+ free(hash_dir);
+ hash_dir = p;
+ }
+
+ if (create_dirs && create_dir(hash_dir) != 0) {
+ cc_log("failed to create %s\n", hash_dir);
+ failed();
+ }
+
+ x_asprintf(&hashname_extra, "%s/%s", hash_dir, hash + nlevels);
+
+ if (set_names)
+ x_asprintf(&hashname, "%s/%s", hash_dir, hash + nlevels);
+
+ free(hash_dir);
+}
+
+
/* find the hash for a command. The hash includes all argument lists,
plus the output from running the compiler with -E */
static void find_hash(ARGS *args)
{
int i;
char *path_stdout, *path_stderr;
- char *hash_dir;
- char *s;
struct stat st;
int status;
- int nlevels = 2;
char *input_base;
char *tmp;
- if ((s = getenv("CCACHE_NLEVELS"))) {
- nlevels = atoi(s);
- if (nlevels < 1) nlevels = 1;
- if (nlevels > 8) nlevels = 8;
- }
-
hash_start();
/* when we are doing the unifying tricks we need to include
@@ -433,25 +479,8 @@
/* we use a N level subdir for the cache path to reduce the impact
on filesystems which are slow for large directories
*/
- s = hash_result();
- x_asprintf(&hash_dir, "%s/%c", cache_dir, s[0]);
- x_asprintf(&stats_file, "%s/stats", hash_dir);
- for (i=1; i<nlevels; i++) {
- char *p;
- if (create_dir(hash_dir) != 0) {
- cc_log("failed to create %s\n", hash_dir);
- failed();
- }
- x_asprintf(&p, "%s/%c", hash_dir, s[i]);
- free(hash_dir);
- hash_dir = p;
- }
- if (create_dir(hash_dir) != 0) {
- cc_log("failed to create %s\n", hash_dir);
- failed();
- }
- x_asprintf(&hashname, "%s/%s", hash_dir, s+nlevels);
- free(hash_dir);
+ hash = hash_result();
+ do_multilevel_name(cache_dir, 1, 1);
}
@@ -461,64 +490,88 @@
otherwise it returns */
static void from_cache(int first)
{
- int fd_stderr, fd_cpp_stderr;
+ int fd_stderr = -1, fd_cpp_stderr;
+ char *dir, **extra;
char *stderr_file;
+ int found = 0;
int ret;
- struct stat st;
- x_asprintf(&stderr_file, "%s.stderr", hashname);
- fd_stderr = open(stderr_file, O_RDONLY | O_BINARY);
- if (fd_stderr == -1) {
- /* it isn't in cache ... */
- free(stderr_file);
- return;
- }
+ for (dir = cache_dir, extra = extra_dirs; dir != 0; dir = *extra++) {
+ struct stat st;
- /* make sure the output is there too */
- if (stat(hashname, &st) != 0) {
- close(fd_stderr);
- unlink(stderr_file);
- free(stderr_file);
- return;
- }
+ do_multilevel_name(dir, 0, 0);
- /* the user might be disabling cache hits */
- if (first && getenv("CCACHE_RECACHE")) {
- close(fd_stderr);
- unlink(stderr_file);
- free(stderr_file);
- return;
+ x_asprintf(&stderr_file, "%s.stderr", hashname_extra);
+ fd_stderr = open(stderr_file, O_RDONLY | O_BINARY);
+ if (fd_stderr == -1) {
+ /* it isn't in cache ... */
+ free(stderr_file);
+ free(hashname_extra);
+ continue;
+ }
+
+ /* make sure the output is there too */
+ if (stat(hashname_extra, &st) != 0) {
+ close(fd_stderr);
+ unlink(stderr_file);
+ free(stderr_file);
+ free(hashname_extra);
+ continue;
+ }
+
+ /* found it */
+
+ /* but the user might be disabling cache hits */
+ if (first && getenv("CCACHE_RECACHE")) {
+ close(fd_stderr);
+ unlink(stderr_file);
+ free(stderr_file);
+ free(hashname_extra);
+ return;
+ }
+
+ /* okay, let's go */
+ utime(stderr_file, NULL);
+ found = 1;
+ break;
}
- utime(stderr_file, NULL);
+ if (!found)
+ return;
if (strcmp(output_file, "/dev/null") == 0) {
ret = 0;
} else {
unlink(output_file);
if (getenv("CCACHE_HARDLINK")) {
- ret = link(hashname, output_file);
+ ret = link(hashname_extra, output_file);
} else {
- ret = copy_file(hashname, output_file);
+ ret = copy_file(hashname_extra, output_file);
}
}
/* the hash file might have been deleted by some external process */
if (ret == -1 && errno == ENOENT) {
cc_log("hashfile missing for %s\n", output_file);
- stats_update(STATS_MISSING);
+ if (dir == cache_dir)
+ stats_update(STATS_MISSING);
close(fd_stderr);
unlink(stderr_file);
+ free(stderr_file);
+ free(hashname_extra);
return;
}
+
free(stderr_file);
+ /* the link might have failed (e.g. cross-device) */
if (ret == -1) {
- ret = copy_file(hashname, output_file);
+ ret = copy_file(hashname_extra, output_file);
if (ret == -1) {
cc_log("failed to copy %s -> %s (%s)\n",
- hashname, output_file, strerror(errno));
- stats_update(STATS_ERROR);
+ hashname_extra, output_file, strerror(errno));
+ if (dir == cache_dir)
+ stats_update(STATS_ERROR);
failed();
}
}
@@ -552,10 +605,17 @@
/* and exit with the right status code */
if (first) {
- cc_log("got cached result for %s\n", output_file);
- stats_update(STATS_CACHED);
+ if (dir == cache_dir) {
+ cc_log("got cached %s from primary cache\n",
+ output_file);
+ stats_update(STATS_CACHED);
+ } else {
+ cc_log("got cached %s from secondary cache %s\n",
+ output_file, dir);
+ }
}
+ free(hashname_extra);
exit(0);
}
@@ -897,7 +957,7 @@
/* run with -E to find the hash */
find_hash(stripped_args);
- /* if we can return from cache at this point then do */
+ /* from_cache calls exit(0) if it finds something */
from_cache(1);
if (getenv("CCACHE_READONLY")) {
@@ -1035,10 +1095,32 @@
temp_dir = cache_dir;
}
+ p = getenv("CCACHE_EXTRA_DIRS");
+
+ if (p) {
+ char *start;
+ int dir;
+ size_t component_len = 0;
+
+ for (start = p, dir = 0; *start != 0 && dir < MAX_DIRS; dir++)
+ {
+ component_len = strcspn(start, ":");
+ x_asprintf(&extra_dirs[dir], "%.*s",
+ (int) component_len, start);
+ start += component_len;
+ start += strspn(start, ":");
+ }
+
+ if (dir == MAX_DIRS && *start != 0) {
+ fprintf(stderr, "ccache: too many components in CCACHE_EXTRA_DIRS\n");
+ exit(1);
+ }
+ }
+
+
cache_logfile = getenv("CCACHE_LOGFILE");
setup_uncached_err();
-
/* the user might have set CCACHE_UMASK */
p = getenv("CCACHE_UMASK");
Index: src/ccache-2.4/ccache.h
===================================================================
--- src.orig/ccache-2.4/ccache.h 2006-12-12 00:44:29.000000000 -0800
+++ src/ccache-2.4/ccache.h 2006-12-13 09:17:08.020103360 -0800
@@ -36,6 +36,9 @@
#define DEFAULT_MAXSIZE (1000*1000)
#endif
+/* maximum number of dirs in the extra path */
+#define MAX_DIRS 32
+
enum stats {
STATS_NONE=0,
STATS_STDOUT,
More information about the ccache
mailing list