[I've originally sent this to the development mailinglist, but since
people appear to be running into bugs that are fixed by this patch,
I thought it might be of interest here as well (also, Duane seems
to be unreachable at the moment). SRB ]
Ok, since development seems to be going fairly fast here, I thought I
share what I have changed so far before it starts to get out of hand.
I tried to stick to the coding style at hand, I hope that it looks
clean enough. The only thing that's easthetically not quite at its
best are the SE (store entry), SED (store entry on disk) and uSE (update
store entry) macros I used. But, since I eventually plan to change most of it
over to using dbz (anyone ever considered that? was it rejected?), this might
not be that much a deal anyway.
Duane, please review these changes, let me know what you think, if
they need reformatting tell me how it has to be changed.
To the others, you might want try it out. It fixes a few bugs, and
adds some sorely needed enhancements to keep the used bandwidth
under control, plus, it makes a start at reducing the memory requirements
(which actually made 3GB cache on a 32MB machine a reality for me).
diff -U 2 -r -d --horizon-lines=3 -u squid-1.1.alpha7/ChangeLog squid-BuGless/ChangeLog
--- squid-1.1.alpha7/ChangeLog Fri Jul 26 23:10:15 1996
+++ squid-BuGless/ChangeLog Sun Jul 28 20:52:18 1996
@@ -1,2 +1,18 @@
+Changes to squid-1.1.alpha7.BuGless:
+
+ - cache_swap 0 is allowed (proxy only server, no swapping), mainly
+ intended for fast maintenance startups where the system already
+ has to be running, but the disk(s) have not been fsck'd yet.
+ - quick_abort actually works now
+ - quick_abort functionality made conditional
+ - ttl_pattern storage wasn't being cleaned upon a reread of
+ the config file (the patterns were added only, got duplicates)
+ - ttl_force_pattern a new config option to override the settings
+ of certain pages despite (or because) of enforced low expiry times
+ - Reduced the struct sentry size. Moved some attributes to mem_obj,
+ eliminated some. On typical 32-bit machines the size was 52 bytes,
+ now it's 32.
+ - Reduced /cache/log file size (still compatible with old format)
+
Changes to squid-1.1.alpha7:
diff -U 2 -r -d --horizon-lines=3 -u squid-1.1.alpha7/include/version.h squid-BuGless/include/version.h
--- squid-1.1.alpha7/include/version.h Fri Jul 26 23:10:28 1996
+++ squid-BuGless/include/version.h Sun Jul 28 20:09:50 1996
@@ -4,4 +4,4 @@
*/
#ifndef SQUID_VERSION
-#define SQUID_VERSION "1.1.alpha7"
+#define SQUID_VERSION "1.1.alpha7.BuGless"
#endif
diff -U 2 -r -d --horizon-lines=3 -u squid-1.1.alpha7/src/cache_cf.c squid-BuGless/src/cache_cf.c
--- squid-1.1.alpha7/src/cache_cf.c Fri Jul 26 21:28:49 1996
+++ squid-BuGless/src/cache_cf.c Sun Jul 28 20:36:46 1996
@@ -165,5 +165,7 @@
#define DefaultCommonLogFormat 0 /* default off */
#define DefaultIdentLookup 0 /* default off */
-#define DefaultQuickAbort 0 /* default off */
+#define DefaultQuickAbortMin -1 /* default off */
+#define DefaultQuickAbortPct 0 /* default off */
+#define DefaultQuickAbortMax 0 /* default off */
#define DefaultNeighborTimeout 2 /* 2 seconds */
#define DefaultStallDelay 1 /* 1 seconds */
@@ -541,10 +543,11 @@
if (sscanf(token, "%d", &pct_age) != 1)
self_destruct();
- }
- token = strtok(NULL, w_space); /* token: age_max */
- if (token != (char *) NULL) { /* age_max is optional */
- if (sscanf(token, "%d", &i) != 1)
- self_destruct();
- age_max = (time_t) (i * 60); /* convert minutes to seconds */
+
+ token = strtok(NULL, w_space); /* token: age_max */
+ if (token != (char *) NULL) { /* age_max is optional */
+ if (sscanf(token, "%d", &i) != 1)
+ self_destruct();
+ age_max = (time_t) (i * 60); /* convert minutes to seconds */
+ }
}
ttlAddToList(pattern, abs_ttl, pct_age, age_max);
@@ -553,4 +556,40 @@
}
+static void parseTTLForcePattern()
+{
+ char *token;
+ char *pattern;
+ time_t abs_ttl = 0;
+ time_t age_max = Config.ageMaxDefault;
+ int i;
+
+ token = strtok(NULL, w_space); /* token: regex pattern */
+ if (token == NULL)
+ self_destruct();
+ pattern = xstrdup(token);
+
+ GetInteger(i); /* token: abs_ttl */
+ abs_ttl = (time_t) (i * 60); /* convert minutes to seconds */
+
+ GetInteger(i);
+ age_max = (time_t) (i * 60); /* convert minutes to seconds */
+ ttlAddToForceList(pattern, abs_ttl, age_max);
+
+ safe_free(pattern);
+}
+
+static void parseQuickAbort()
+{
+ char *token;
+ int i;
+
+ GetInteger(i);
+ Config.quickAbort.min = i * 1024;
+ GetInteger(i);
+ Config.quickAbort.pct = i * 128 / 100; /* 128 is full scale */
+ GetInteger(i);
+ Config.quickAbort.max = i * 1024;
+}
+
static void parseNegativeLine()
{
@@ -1115,4 +1154,10 @@
parseTTLPattern();
+ else if (!strcmp(token, "ttl_force_pattern"))
+ parseTTLForcePattern();
+
+ else if (!strcmp(token, "quick_abort"))
+ parseQuickAbort();
+
else if (!strcmp(token, "negative_ttl"))
parseNegativeLine();
@@ -1163,7 +1208,4 @@
parseOnOff(&Config.sourcePing);
- else if (!strcmp(token, "quick_abort"))
- parseOnOff(&Config.quickAbort);
-
else if (!strcmp(token, "emulate_httpd_log"))
parseOnOff(&Config.commonLogFormat);
@@ -1269,6 +1311,4 @@
printf(" This will cause serious problems with your cache!!!\n");
printf(" Change your configuration file.\n");
- Config.Swap.maxSize = Config.Mem.maxSize >> 10;
- printf(" For this run, however, %s will use %d kbytes for cache_swap.\n", appname, Config.Swap.maxSize);
fflush(stdout); /* print message */
}
@@ -1362,4 +1402,5 @@
wordlistDestroy(&Config.dns_testname_list);
safe_free(Config.sslProxy.host);
+ ttlFreeList();
}
@@ -1399,5 +1440,7 @@
Config.hotVmFactor = DefaultHotVmFactor;
Config.sourcePing = DefaultSourcePing;
- Config.quickAbort = DefaultQuickAbort;
+ Config.quickAbort.min = DefaultQuickAbortMin;
+ Config.quickAbort.pct = DefaultQuickAbortPct;
+ Config.quickAbort.max = DefaultQuickAbortMax;
Config.commonLogFormat = DefaultCommonLogFormat;
Config.debugOptions = safe_xstrdup(DefaultDebugOptions);
diff -U 2 -r -d --horizon-lines=3 -u squid-1.1.alpha7/src/cache_cf.h squid-BuGless/src/cache_cf.h
--- squid-1.1.alpha7/src/cache_cf.h Fri Jul 26 21:28:49 1996
+++ squid-BuGless/src/cache_cf.h Sun Jul 28 01:41:54 1996
@@ -153,4 +153,9 @@
u_short relayPort;
} Wais;
+ struct {
+ int min;
+ int pct;
+ int max;
+ } quickAbort;
int negativeTtl;
int negativeDnsTtl;
@@ -188,5 +193,4 @@
int redirectChildren;
int sourcePing;
- int quickAbort;
int commonLogFormat;
int identLookup;
diff -U 2 -r -d --horizon-lines=3 -u squid-1.1.alpha7/src/icp.c squid-BuGless/src/icp.c
--- squid-1.1.alpha7/src/icp.c Fri Jul 26 23:09:34 1996
+++ squid-BuGless/src/icp.c Sun Jul 28 15:46:33 1996
@@ -379,5 +379,6 @@
debug(12, 5, "icpSendMoreData: <URL:%s> sz %d: len %d: off %d.\n",
entry->url,
- entry->object_len,
+ entry->mem_obj->object_len,
+/* test really needed? SRB */
entry->mem_obj ? entry->mem_obj->e_current_len : 0,
icpState->offset);
@@ -441,8 +442,8 @@
entry = icpState->entry;
- debug(12, 5, "icpHandleStoreComplete: FD %d: sz %d: err %d: off %d: len %d: tsmp %d: lref %d.\n",
+ debug(12, 5, "icpHandleStoreComplete: FD %d: sz %d: err %d: off %d: len: %d: lref %d.\n",
fd, size, errflag,
- icpState->offset, entry->object_len,
- entry->timestamp, entry->lastref);
+ icpState->offset, SE(entry,object_len),
+ SE(entry,lastref));
if (errflag) {
/* if runs in quick abort mode, set flag to tell
@@ -458,5 +459,5 @@
/* More data available locally; write it now */
icpSendMoreData(fd, icpState);
- } else if (icpState->offset == entry->object_len &&
+ } else if (icpState->offset == SE(entry,object_len) &&
entry->store_status != STORE_PENDING) {
/* We're finished case */
@@ -517,4 +518,6 @@
/* XXX: Should we ignore this? */
if (mem->reply->code != 200) {
+ debug(12, 4, "icpGetHeadersForIMS: Reply code %d!=200\n",
+ mem->reply->code );
put_free_8k_page(icpState->buf);
icpState->buf = NULL;
@@ -537,5 +540,5 @@
date = parse_rfc850(mem->reply->date);
else
- date = entry->timestamp;
+ date = SED(entry,timestamp);
/* Find size of the object */
@@ -543,5 +546,5 @@
length = mem->reply->content_length;
else
- length = entry->object_len - mime_headers_size(icpState->buf);
+ length = SE(entry,object_len) - mime_headers_size(icpState->buf);
put_free_8k_page(icpState->buf);
@@ -680,5 +683,4 @@
if (entry != NULL) {
CacheInfo->proto_hit(CacheInfo, icpState->request->protocol);
- entry->refcount++;
} else {
CacheInfo->proto_miss(CacheInfo, icpState->request->protocol);
@@ -755,5 +757,4 @@
/* NOTE, don't call storeLockObject(), storeCreateEntry() does it */
- entry->refcount++; /* MISS CASE */
entry->mem_obj->fd_of_first_client = fd;
fd_table[fd].store_entry = entry;
@@ -933,5 +934,6 @@
debug(12, 3, "icpUdpSendEntry: entry = %p\n", entry);
- buf_len = sizeof(icp_common_t) + strlen(url) + 1 + 2 + entry->object_len;
+ buf_len = sizeof(icp_common_t) + strlen(url) + 1 + 2 +
+ SE(entry,object_len);
#ifdef CHECK_BAD_ADDRS
@@ -958,12 +960,12 @@
urloffset = buf + sizeof(icp_common_t);
xmemcpy(urloffset, url, strlen(url));
- data_sz = htons((u_short) entry->object_len);
+ data_sz = htons((u_short) SE(entry,object_len));
entryoffset = urloffset + strlen(url) + 1;
xmemcpy(entryoffset, &data_sz, sizeof(u_short));
entryoffset += sizeof(u_short);
- size = m->data->mem_copy(m->data, 0, entryoffset, entry->object_len);
- if (size != entry->object_len) {
+ size = m->data->mem_copy(m->data, 0, entryoffset, m->object_len);
+ if (size != m->object_len) {
debug(12, 1, "icpUdpSendEntry: copy failed, wanted %d got %d bytes\n",
- entry->object_len, size);
+ m->object_len, size);
safe_free(buf);
return;
@@ -1079,5 +1081,6 @@
(entry->store_status == STORE_OK) &&
(entry->expires > (squid_curtime + Config.negativeTtl))) {
- pkt_len = sizeof(icp_common_t) + strlen(url) + 1 + 2 + entry->object_len;
+ pkt_len = sizeof(icp_common_t) + strlen(url) + 1 + 2 +
+ SE(entry,object_len);
if (header.flags & ICP_FLAG_HIT_OBJ && pkt_len < SQUID_UDP_SO_SNDBUF) {
icpHitObjState = xcalloc(1, sizeof(icpHitObjStateData));
@@ -1797,12 +1800,26 @@
if (icpState->entry == NULL)
return;
- if (icpState->entry->lock_count != 1)
+ if (icpState->entry->lock_count > 2)
return;
if (icpState->entry->store_status == STORE_OK)
return;
- if (!Config.quickAbort &&
- BIT_TEST(icpState->flags, REQ_CACHABLE) &&
- !BIT_TEST(icpState->entry->flag, KEY_PRIVATE))
- return;
+ if (BIT_TEST(icpState->flags, REQ_CACHABLE) &&
+ !BIT_TEST(icpState->entry->flag, KEY_PRIVATE)) {
+
+ long totallen = icpState->entry->mem_obj->reply->content_length;
+
+ long curlen = icpState->entry->mem_obj->e_current_len;
+ long minlen = Config.quickAbort.min;
+
+ if (minlen >= 0 && totallen > curlen) {
+ if ((totallen-curlen) < minlen)
+ return;
+ if ((totallen-curlen) <= Config.quickAbort.max &&
+ curlen/(totallen/128U) > Config.quickAbort.pct)
+ return;
+ debug(12, 3, "CheckQuickAbort: aborting %ld (%ld)\n",
+ curlen, totallen);
+ }
+ }
BIT_SET(icpState->entry->flag, CLIENT_ABORT_REQUEST);
storeReleaseRequest(icpState->entry);
@@ -1827,5 +1844,6 @@
return;
}
- if (n == 0 && entry != NULL && icpState->offset == entry->object_len &&
+ if (n == 0 && entry != NULL &&
+ icpState->offset == SE(entry,object_len) &&
entry->store_status != STORE_PENDING) {
/* All data has been delivered */
diff -U 2 -r -d --horizon-lines=3 -u squid-1.1.alpha7/src/neighbors.c squid-BuGless/src/neighbors.c
--- squid-1.1.alpha7/src/neighbors.c Fri Jul 26 23:09:37 1996
+++ squid-BuGless/src/neighbors.c Sun Jul 28 01:31:05 1996
@@ -681,5 +681,5 @@
debug(15, 0, "Ignoring ICP_OP_HIT_OBJ from non-neighbor %s\n",
inet_ntoa(from->sin_addr));
- } else if (entry->object_len != 0) {
+ } else if (mem->object_len != 0) {
debug(15, 1, "Too late UDP_HIT_OBJ '%s'?\n", entry->url);
} else {
diff -U 2 -r -d --horizon-lines=3 -u squid-1.1.alpha7/src/squid.h squid-BuGless/src/squid.h
--- squid-1.1.alpha7/src/squid.h Fri Jul 26 18:59:06 1996
+++ squid-BuGless/src/squid.h Sun Jul 28 01:07:26 1996
@@ -288,4 +288,6 @@
extern char *storeToString _PARAMS((StoreEntry *));
extern void ttlSet _PARAMS((StoreEntry *));
+extern void ttlFreeList _PARAMS((void));
extern void ttlAddToList _PARAMS((char *, time_t, int, time_t));
+extern void ttlAddToForceList _PARAMS((char *, time_t, time_t));
extern int waisStart _PARAMS((int, char *, method_t, char *, StoreEntry *));
diff -U 2 -r -d --horizon-lines=3 -u squid-1.1.alpha7/src/stat.c squid-BuGless/src/stat.c
--- squid-1.1.alpha7/src/stat.c Fri Jul 26 19:18:26 1996
+++ squid-BuGless/src/stat.c Sun Jul 28 15:47:29 1996
@@ -159,5 +159,4 @@
p->hit = 0;
p->miss = 0;
- p->refcount = 0;
p->transferbyte = 0;
@@ -174,5 +173,4 @@
p->hit += q->hit;
p->miss += q->miss;
- p->refcount += q->refcount;
p->transferbyte += q->transferbyte;
}
@@ -187,5 +185,5 @@
(float) p->miss);
}
- storeAppendPrintf(sentry, "{%8.8s %d %d %d %d %4.2f %d %d %d}\n",
+ storeAppendPrintf(sentry, "{%8.8s %d %d %d %d %4.2f %d 0 %d}\n",
p->protoname,
p->object_count,
@@ -195,5 +193,4 @@
p->hitratio,
(secs ? p->transferbyte / secs : 0),
- p->refcount,
p->transferbyte);
}
@@ -280,17 +277,16 @@
debug(18, 3, "stat_objects_get: Processed %d objects...\n", N);
}
- obj_size = entry->object_len;
npend = storePendingNClients(entry);
- if (entry->mem_obj)
- obj_size = entry->mem_obj->e_current_len;
- storeAppendPrintf(sentry, "{ %s %d %s %s %s %s %d %d %s %s }\n",
+ obj_size = entry->mem_obj ?
+ entry->mem_obj->e_current_len :
+ SE(entry,object_len);
+ storeAppendPrintf(sentry, "{ %s %d %s %s %s %s %d 0 %s %s }\n",
entry->url,
obj_size,
- elapsed_time(entry, (int) entry->timestamp, space),
+ elapsed_time(entry, (int) SED(entry,timestamp), space),
flags_describe(entry),
- elapsed_time(entry, (int) entry->lastref, space2),
+ elapsed_time(entry, (int) SE(entry,lastref), space2),
ttl_describe(entry),
npend,
- (int) entry->refcount,
mem_describe(entry),
stat_describe(entry));
diff -U 2 -r -d --horizon-lines=3 -u squid-1.1.alpha7/src/store.c squid-BuGless/src/store.c
--- squid-1.1.alpha7/src/store.c Fri Jul 26 23:09:40 1996
+++ squid-BuGless/src/store.c Sun Jul 28 20:35:53 1996
@@ -513,5 +513,5 @@
fatal_dump(NULL);
}
- e->lastref = squid_curtime;
+ uSE(e,lastref,squid_curtime);
/* If the object is NOT_IN_MEMORY, fault it in. */
@@ -577,8 +577,7 @@
e->lock_count++;
destroy_MemObjectData(e->mem_obj);
- e->object_len = 0;
e->mem_obj->data = new_MemObjectData();
storeAppend(e, e->mem_obj->e_abort_msg, strlen(e->mem_obj->e_abort_msg));
- e->object_len = e->mem_obj->e_current_len
+ e->mem_obj->object_len = e->mem_obj->e_current_len
= strlen(e->mem_obj->e_abort_msg);
BIT_RESET(e->flag, ABORT_MSG_PENDING);
@@ -777,7 +776,5 @@
e->swap_file_number = -1;
mem->data = new_MemObjectData();
- e->refcount = 0;
- e->lastref = squid_curtime;
- e->timestamp = 0; /* set in storeSwapOutHandle() */
+ mem->lastref = squid_curtime;
e->ping_status = PING_NONE;
@@ -798,5 +795,5 @@
/* Add a new object to the cache with empty memory copy and pointer to disk
* use to rebuild store from disk. */
-StoreEntry *storeAddDiskRestore(url, file_number, size, expires, timestamp, lastmod)
+StoreEntry *storeAddDiskRestore(url, file_number, size, expires, timestamp)
char *url;
int file_number;
@@ -804,5 +801,4 @@
time_t expires;
time_t timestamp;
- time_t lastmod;
{
StoreEntry *e = NULL;
@@ -828,12 +824,9 @@
e->swap_file_number = file_number;
file_map_bit_set(file_number);
- e->object_len = size;
e->lock_count = 0;
+/* object_len */
BIT_RESET(e->flag, CLIENT_ABORT_REQUEST);
- e->refcount = 0;
- e->lastref = squid_curtime;
- e->timestamp = timestamp;
+/* e->lastref = squid_curtime; */
e->expires = expires;
- e->lastmod = lastmod;
e->ping_status = PING_NONE;
return e;
@@ -1113,4 +1106,48 @@
}
+static struct stat*storeGet_FSE(entry)
+ StoreEntry *entry;
+{
+ static int swap_file_number = -1;
+ static struct stat stbuf;
+
+ if ((entry->swap_file_number != swap_file_number ||
+ swap_file_number == -1) &&
+ (entry->swap_file_number == -1 ||
+ stat(storeSwapFullPath(entry->swap_file_number, (char*)0), &stbuf))) {
+
+ stbuf.st_size = 0;
+ if (entry->mem_obj &&
+ !(stbuf.st_size = entry->mem_obj->object_len))
+ stbuf.st_size = entry->mem_obj->e_current_len;
+ stbuf.st_mtime = stbuf.st_atime = squid_curtime;
+ swap_file_number = -1;
+ } else {
+ swap_file_number = entry->swap_file_number;
+ if (entry->mem_obj)
+ entry->mem_obj->object_len = stbuf.st_size;
+ }
+
+ return &stbuf;
+}
+
+time_t SED_timestamp(entry)
+ StoreEntry *entry;
+{
+ return storeGet_FSE(entry)->st_mtime;
+}
+
+time_t SED_lastref(entry)
+ StoreEntry *entry;
+{
+ return storeGet_FSE(entry)->st_atime;
+}
+
+long SED_object_len(entry)
+ StoreEntry *entry;
+{
+ return storeGet_FSE(entry)->st_size;
+}
+
/* swapping in handle */
int storeSwapInHandle(fd_notused, buf, len, flag, e, offset_notused)
@@ -1141,5 +1178,5 @@
debug(20, 5, "storeSwapInHandle: len = %d\n", len);
debug(20, 5, "storeSwapInHandle: e->e_current_len = %d\n", mem->e_current_len);
- debug(20, 5, "storeSwapInHandle: e->object_len = %d\n", e->object_len);
+ debug(20, 5, "storeSwapInHandle: e->object_len = %d\n", mem->object_len);
/* always call these, even if len == 0 */
@@ -1147,5 +1184,5 @@
storeAppend(e, buf, len);
- if (mem->e_current_len < e->object_len && flag != DISK_EOF) {
+ if (mem->e_current_len < SE(e,object_len) && flag != DISK_EOF) {
/* some more data to swap in, reschedule */
file_read(mem->swapin_fd,
@@ -1163,12 +1200,5 @@
debug(20, 5, "storeSwapInHandle: SwapIn complete: <URL:%s> from %s.\n",
e->url, storeSwapFullPath(e->swap_file_number, NULL));
- if (mem->e_current_len != e->object_len) {
- debug(20, 0, "storeSwapInHandle: WARNING! Object size mismatch.\n");
- debug(20, 0, " --> <URL:%s>\n", e->url);
- debug(20, 0, " --> Expecting %d bytes from file: %s\n", e->object_len,
- storeSwapFullPath(e->swap_file_number, NULL));
- debug(20, 0, " --> Only read %d bytes\n",
- mem->e_current_len);
- }
+ mem->object_len = mem->e_current_len;
if (mem->swapin_complete_handler) {
(*mem->swapin_complete_handler) (0, mem->swapin_complete_data);
@@ -1202,4 +1232,5 @@
/* create additional structure for object in memory */
e->mem_obj = mem = new_MemObject();
+ mem->object_len = SED(e,object_len);
path = storeSwapFullPath(e->swap_file_number, NULL);
@@ -1236,10 +1267,7 @@
LOCAL_ARRAY(char, logmsg, MAX_URL << 1);
/* Note this printf format appears in storeWriteCleanLog() too */
- sprintf(logmsg, "%08x %08x %08x %08x %9d %s\n",
+ sprintf(logmsg, "%08x 0 %08x 0 0 %s\n",
(int) e->swap_file_number,
- (int) e->timestamp,
(int) e->expires,
- (int) e->lastmod,
- e->object_len,
e->url);
file_write(swaplog_fd,
@@ -1262,5 +1290,4 @@
debug(20, 3, "storeSwapOutHandle: '%s'\n", e->key);
- e->timestamp = squid_curtime;
storeSwapFullPath(e->swap_file_number, filename);
@@ -1292,5 +1319,5 @@
mem->e_swap_buf_len);
debug(20, 6, "storeSwapOutHandle: e->object_len = %d\n",
- e->object_len);
+ mem->object_len);
debug(20, 6, "storeSwapOutHandle: store_swap_size = %dk\n",
store_swap_size);
@@ -1299,5 +1326,5 @@
/* round up */
store_swap_size += ((mem->e_swap_buf_len + 1023) >> 10);
- if (mem->swap_offset >= e->object_len) {
+ if (mem->swap_offset >= mem->object_len) {
/* swapping complete */
e->swap_status = SWAP_OK;
@@ -1310,5 +1337,5 @@
CacheInfo->proto_newobject(CacheInfo,
mem->request->protocol,
- e->object_len,
+ mem->object_len,
FALSE);
/* check if it's request to be released. */
@@ -1401,6 +1428,4 @@
StoreEntry *e = NULL;
time_t expires;
- time_t timestamp;
- time_t lastmod;
int scan1;
int scan2;
@@ -1413,4 +1438,5 @@
int count;
int x;
+ time_t timestamp;
/* load a number of objects per invocation */
@@ -1436,8 +1462,8 @@
x = sscanf(data->line_in, "%x %x %x %x %d %s",
&sfileno, /* swap_file_number */
- &scan1, /* timestamp */
+ &scan1, /* zip srb */
&scan2, /* expires */
- &scan3, /* last modified */
- &scan4, /* size */
+ &scan3, /* zip srb */
+ &scan4, /* zip srb */
url); /* url */
if (x > 0)
@@ -1448,21 +1474,20 @@
continue;
}
- timestamp = (time_t) scan1;
expires = (time_t) scan2;
- lastmod = (time_t) scan3;
- size = (off_t) scan4;
+
+ if (stat(swapfile, &sb) < 0) {
+ debug(20, 3, "storeRebuildFromDisk: Swap file missing: <URL:%s>: %s: %s.\n", url, swapfile, xstrerror());
+ if (opt_unlink_on_reload)
+ safeunlink(swapfile, 1);
+ continue;
+ }
+ size = sb.st_size;
if (store_rebuilding != STORE_REBUILDING_FAST) {
- if (stat(swapfile, &sb) < 0) {
- if (expires < squid_curtime) {
- debug(20, 3, "storeRebuildFromDisk: Expired: <URL:%s>\n", url);
- if (opt_unlink_on_reload)
- safeunlink(swapfile, 1);
- data->expcount++;
- } else {
- debug(20, 3, "storeRebuildFromDisk: Swap file missing: <URL:%s>: %s: %s.\n", url, swapfile, xstrerror());
- if (opt_unlink_on_reload)
- safeunlink(swapfile, 1);
- }
+ if (expires < squid_curtime) {
+ debug(20, 3, "storeRebuildFromDisk: Expired: <URL:%s>\n", url);
+ if (opt_unlink_on_reload)
+ safeunlink(swapfile, 1);
+ data->expcount++;
continue;
}
@@ -1473,17 +1498,4 @@
continue;
}
- /* timestamp might be a little bigger than sb.st_mtime */
- delta = (int) (timestamp - sb.st_mtime);
- if (delta > REBUILD_TIMESTAMP_DELTA_MAX || delta < 0) {
- /* this log entry doesn't correspond to this file */
- data->clashcount++;
- continue;
- }
- /* Wrong size? */
- if (sb.st_size != size) {
- /* this log entry doesn't correspond to this file */
- data->clashcount++;
- continue;
- }
timestamp = sb.st_mtime;
debug(20, 9, "storeRebuildFromDisk: swap file exists: <URL:%s>: %s\n",
@@ -1491,5 +1503,5 @@
}
if ((e = storeGet(url))) {
- if (e->timestamp > timestamp) {
+ if (SED(e, timestamp) > timestamp) {
/* already have a newer object in memory, throw old one away */
debug(20, 3, "storeRebuildFromDisk: Replaced: %s\n", url);
@@ -1529,6 +1541,5 @@
(int) size,
expires,
- timestamp,
- lastmod);
+ timestamp);
storeSwapLog(e);
CacheInfo->proto_newobject(CacheInfo,
@@ -1671,5 +1682,5 @@
} else if (!storeEntryValidLength(e)) {
debug(20, 2, "storeCheckSwapable: NO: wrong content-length\n");
- } else
+ } else if (Config.Swap.maxSize > 0)
return 1;
@@ -1687,7 +1698,7 @@
debug(20, 3, "storeComplete: '%s'\n", e->key);
- e->object_len = e->mem_obj->e_current_len;
+ e->mem_obj->object_len = e->mem_obj->e_current_len;
InvokeHandlers(e);
- e->lastref = squid_curtime;
+ e->mem_obj->lastref = squid_curtime;
e->store_status = STORE_OK;
storeSetMemStatus(e, IN_MEMORY);
@@ -1718,5 +1729,5 @@
/* No DISK swap for negative cached object */
e->swap_status = NO_SWAP;
- e->lastref = squid_curtime;
+ e->mem_obj->lastref = squid_curtime;
/* In case some parent responds late and
* tries to restart the fetch, say that it's been
@@ -1750,5 +1761,5 @@
/* We assign an object length here--The only other place we assign the
* object length is in storeComplete() */
- e->object_len = e->mem_obj->e_current_len;
+ e->mem_obj->object_len = e->mem_obj->e_current_len;
/* Call handlers so they can report error. */
@@ -1841,7 +1852,4 @@
debug(20, 5, "removeOldEntry: Checking: %s\n", e->url);
debug(20, 6, "removeOldEntry: * curtime: %8ld\n", curtime);
- debug(20, 6, "removeOldEntry: * e->timestamp: %8ld\n", e->timestamp);
- debug(20, 6, "removeOldEntry: * time in cache: %8ld\n",
- curtime - e->timestamp);
debug(20, 6, "removeOldEntry: * time-to-live: %8ld\n",
e->expires - squid_curtime);
@@ -2064,14 +2072,13 @@
StoreEntry **e1, **e2;
{
+ StoreEntry *s1,*s2;
+ time_t dif;
+
if (!e1 || !e2)
fatal_dump(NULL);
+ s1 = *e1; s2 = *e2;
+ dif = SE(s1,lastref) - SE(s2,lastref);
- if ((*e1)->lastref > (*e2)->lastref)
- return (1);
-
- if ((*e1)->lastref < (*e2)->lastref)
- return (-1);
-
- return (0);
+ return dif > 0 ? 1 : dif < 0 ? -1 : 0;
}
@@ -2114,5 +2121,5 @@
dynamic_array *LRU_list;
hash_link *link_ptr = NULL, *next = NULL;
- unsigned int kb_size = ((size + 1023) >> 10);
+ int kb_size = ((size + 1023) >> 10);
if (store_swap_size + kb_size <= store_swap_low)
@@ -2163,5 +2170,5 @@
} else {
debug(20, 2, "storeGetSwapSpace: Can't purge %7d bytes: <URL:%s>\n",
- e->object_len, e->url);
+ e->mem_obj->object_len, e->url);
if (e->lock_count) {
debug(20, 2, "\t\te->lock_count %d\n", e->lock_count);
@@ -2326,11 +2333,14 @@
if (e->swap_status == SWAP_OK && (e->swap_file_number > -1)) {
+ long object_len;
+
+ object_len = SE(e,object_len);
(void) safeunlink(storeSwapFullPath(e->swap_file_number, NULL), 1);
file_map_bit_reset(e->swap_file_number);
e->swap_file_number = -1;
- store_swap_size -= (e->object_len + 1023) >> 10;
+ store_swap_size -= (object_len + 1023) >> 10;
CacheInfo->proto_purgeobject(CacheInfo,
urlParseProtocol(e->url),
- e->object_len);
+ object_len);
}
storeHashDelete(e);
@@ -2552,5 +2562,6 @@
debug(20, 3, "storeEntryValidLength: Checking '%s'\n", e->key);
- debug(20, 5, "storeEntryValidLength: object_len = %d\n", e->object_len);
+ debug(20, 5, "storeEntryValidLength: object_len = %d\n",
+ e->mem_obj->object_len);
debug(20, 5, "storeEntryValidLength: hdr_sz = %d\n", hdr_sz);
debug(20, 5, "storeEntryValidLength: content_length = %d\n", content_length);
@@ -2566,5 +2577,5 @@
return 1;
}
- diff = hdr_sz + content_length - e->object_len;
+ diff = hdr_sz + content_length - SE(e,object_len);
if (diff != 0) {
debug(20, 3, "storeEntryValidLength: %d bytes too %s; '%s'\n",
@@ -2670,24 +2681,33 @@
storeSanityCheck();
file_map_create(MAX_SWAP_FILE);
- dir_created = storeVerifySwapDirs(opt_zap_disk_store);
+
+ if (Config.Swap.maxSize > 0)
+ dir_created = storeVerifySwapDirs(opt_zap_disk_store);
+
storeCreateHashTable(urlcmp);
- sprintf(swaplog_file, "%s/log", swappath(0));
+ if (Config.Swap.maxSize <= 0) {
+ swaplog_fd = -1;
+ store_rebuilding = STORE_NO_STORE;
+ } else {
+ sprintf(swaplog_file, "%s/log", swappath(0));
- swaplog_fd = file_open(swaplog_file, NULL, O_WRONLY | O_CREAT);
- debug(20, 3, "swaplog_fd %d is now '%s'\n", swaplog_fd, swaplog_file);
- if (swaplog_fd < 0) {
- sprintf(tmp_error_buf, "Cannot open swap logfile: %s", swaplog_file);
- fatal(tmp_error_buf);
- }
- swaplog_lock = file_write_lock(swaplog_fd);
+ swaplog_fd = file_open(swaplog_file, NULL, O_WRONLY | O_CREAT);
+ debug(20, 3, "swaplog_fd %d is now '%s'\n", swaplog_fd, swaplog_file);
+ if (swaplog_fd < 0) {
+ sprintf(tmp_error_buf, "Cannot open swap logfile: %s",
+ swaplog_file);
+ fatal(tmp_error_buf);
+ }
+ swaplog_lock = file_write_lock(swaplog_fd);
- if (!opt_zap_disk_store)
- storeStartRebuildFromDisk();
- else
- store_rebuilding = STORE_NOT_REBUILDING;
+ if (!opt_zap_disk_store)
+ storeStartRebuildFromDisk();
+ else
+ store_rebuilding = STORE_NOT_REBUILDING;
- if (dir_created || opt_zap_disk_store)
- storeCreateSwapSubDirs();
+ if (dir_created || opt_zap_disk_store)
+ storeCreateSwapSubDirs();
+ }
store_mem_high = (long) (Config.Mem.maxSize / 100) *
@@ -2769,5 +2789,6 @@
/* We can't delete objects while rebuilding swap */
- if (store_rebuilding == STORE_REBUILDING_FAST)
+ if (store_rebuilding == STORE_REBUILDING_FAST ||
+ store_rebuilding == STORE_NO_STORE)
return -1;
@@ -2835,13 +2856,11 @@
if (e->swap_status != SWAP_OK)
continue;
- if (e->object_len <= 0)
- continue;
storeSwapFullPath(e->swap_file_number, swapfilename);
x = fprintf(fp, "%08x %08x %08x %08x %9d %s\n",
(int) e->swap_file_number,
- (int) e->timestamp,
+ 0,
(int) e->expires,
- (int) e->lastmod,
- e->object_len,
+ 0,
+ 0,
e->url);
if (x < 0) {
diff -U 2 -r -d --horizon-lines=3 -u squid-1.1.alpha7/src/store.h squid-BuGless/src/store.h
--- squid-1.1.alpha7/src/store.h Fri Jul 26 19:00:37 1996
+++ squid-BuGless/src/store.h Sun Jul 28 15:58:33 1996
@@ -171,4 +171,6 @@
int e_current_len;
+ int object_len;
+ time_t lastref;
/* The lowest offset that store keep VM copy around
* use for "delete_behind" mechanism for a big object */
@@ -232,11 +234,6 @@
u_num32 flag;
- u_num32 timestamp;
- u_num32 refcount;
- time_t lastref;
time_t expires;
- time_t lastmod;
- int object_len;
int swap_file_number;
@@ -254,4 +251,20 @@
};
+#define uSE(entry,what,value) \
+ ((entry)->mem_obj ? \
+ ((entry)->mem_obj->what)=(value) : \
+ 0)
+
+#define SE(entry,what) \
+ ((entry)->mem_obj && ((entry)->mem_obj->what) ? \
+ ((entry)->mem_obj->what) : \
+ SED(entry,what))
+
+#define SED(entry,what) (SED_##what(entry))
+
+time_t SED_timestamp _PARAMS((struct sentry*entry));
+time_t SED_lastref _PARAMS((struct sentry*entry));
+long SED_object_len _PARAMS((struct sentry*entry));
+
/* ----------------------------------------------------------------- */
@@ -323,4 +336,5 @@
#define STORE_REBUILDING_SLOW 1
#define STORE_REBUILDING_FAST 2
+#define STORE_NO_STORE 3
#endif
diff -U 2 -r -d --horizon-lines=3 -u squid-1.1.alpha7/src/storetoString.c squid-BuGless/src/storetoString.c
--- squid-1.1.alpha7/src/storetoString.c Fri Jul 26 19:00:37 1996
+++ squid-BuGless/src/storetoString.c Sun Jul 28 16:00:49 1996
@@ -157,20 +157,15 @@
strcat(stsbuf, tmpbuf);
- t = (time_t) e->timestamp;
- sprintf(tmpbuf, "Timestamp: %9d [%s]\n", (int) e->timestamp,
- mkhttpdlogtime(&t));
- strcat(stsbuf, tmpbuf);
-
- t = (time_t) e->lastref;
- sprintf(tmpbuf, "Lastref : %9d [%s]\n", (int) e->lastref,
+ t = (time_t) SE(e,lastref);
+ sprintf(tmpbuf, "Lastref : %9d [%s]\n", (int) t,
mkhttpdlogtime(&t));
strcat(stsbuf, tmpbuf);
t = (time_t) e->expires;
- sprintf(tmpbuf, "Expires : %9d [%s]\n", (int) e->expires,
+ sprintf(tmpbuf, "Expires : %9d [%s]\n", (int) t,
mkhttpdlogtime(&t));
strcat(stsbuf, tmpbuf);
- sprintf(tmpbuf, "ObjectLen: %d\n", e->object_len);
+ sprintf(tmpbuf, "ObjectLen: %d\n", SE(e,object_len));
strcat(stsbuf, tmpbuf);
@@ -193,7 +188,4 @@
strcat(stsbuf, tmpbuf);
- sprintf(tmpbuf, "RefCount: %u\n", e->refcount);
- strcat(stsbuf, tmpbuf);
-
sprintf(tmpbuf, "LockCount: %d\n", e->lock_count);
strcat(stsbuf, tmpbuf);
diff -U 2 -r -d --horizon-lines=3 -u squid-1.1.alpha7/src/ttl.c squid-BuGless/src/ttl.c
--- squid-1.1.alpha7/src/ttl.c Thu Jul 25 09:10:45 1996
+++ squid-BuGless/src/ttl.c Sun Jul 28 01:18:44 1996
@@ -125,4 +125,6 @@
static ttl_t *TTL_tbl = NULL;
static ttl_t *TTL_tail = NULL;
+static ttl_t *TTL_tbl_force = NULL;
+static ttl_t *TTL_tail_force = NULL;
#define TTL_EXPIRES 0x01
@@ -133,4 +135,25 @@
#define TTL_ABSOLUTE 0x20
#define TTL_DEFAULT 0x40
+#define TTL_FORCE 0x80
+
+static void ttlFreeListgeneric(t)
+ ttl_t *t;
+{
+ ttl_t *tnext;
+
+ for (; t; t = tnext) {
+ tnext = t->next;
+ safe_free(t->pattern);
+ regfree(&t->compiled_pattern);
+ safe_free(t);
+ }
+}
+
+void ttlFreeList()
+{
+ ttlFreeListgeneric(TTL_tbl);
+ ttlFreeListgeneric(TTL_tbl_force);
+ TTL_tail = TTL_tbl = TTL_tail_force = TTL_tbl_force = 0;
+}
void ttlAddToList(pattern, abs_ttl, pct_age, age_max)
@@ -148,5 +171,5 @@
return;
}
- pct_age = pct_age < 0 ? 0 : pct_age > 100 ? 100 : pct_age;
+ pct_age = pct_age < 0 ? 0 : pct_age;
age_max = age_max < 0 ? 0 : age_max;
@@ -166,4 +189,33 @@
}
+void ttlAddToForceList(pattern, abs_ttl, age_max)
+ char *pattern;
+ time_t abs_ttl;
+ time_t age_max;
+{
+ ttl_t *t;
+ regex_t comp;
+
+ if (regcomp(&comp, pattern, REG_EXTENDED) != REG_NOERROR) {
+ debug(22, 0, "ttlAddToList: Invalid regular expression: %s\n",
+ pattern);
+ return;
+ }
+ age_max = age_max < 0 ? 0 : age_max;
+
+ t = xcalloc(1, sizeof(ttl_t));
+ t->pattern = (char *) xstrdup(pattern);
+ t->compiled_pattern = comp;
+ t->abs_ttl = abs_ttl;
+ t->age_max = age_max;
+ t->next = NULL;
+
+ if (!TTL_tbl_force)
+ TTL_tbl_force = t;
+ if (TTL_tail_force)
+ TTL_tail_force->next = t;
+ TTL_tail_force = t;
+}
+
@@ -223,16 +275,5 @@
if (expire > -1) {
ttl = (expire - squid_curtime);
- debug(22, 4, "ttlSet: [%c%c%c%c%c%c%c] %6.2lf days %s\n",
- flags & TTL_EXPIRES ? 'E' : '.',
- flags & TTL_SERVERDATE ? 'S' : '.',
- flags & TTL_LASTMOD ? 'L' : '.',
- flags & TTL_MATCHED ? 'M' : '.',
- flags & TTL_PCTAGE ? 'P' : '.',
- flags & TTL_ABSOLUTE ? 'A' : '.',
- flags & TTL_DEFAULT ? 'D' : '.',
- (double) ttl / 86400, entry->url);
- entry->expires = expire;
- entry->lastmod = last_modified > -1 ? last_modified : served_date;
- return;
+ goto finalcheck;
}
/* Calculate default TTL for later use */
@@ -282,5 +323,23 @@
}
- debug(22, 4, "ttlSet: [%c%c%c%c%c%c%c] %6.2lf days %s\n",
+finalcheck:
+ if (flags & (TTL_EXPIRES|TTL_PCTAGE)) {
+ match = NULL;
+ for (t = TTL_tbl_force; t; t = t->next) {
+ if (t->age_max >= ttl &&
+ regexec(&(t->compiled_pattern), entry->url, 0, 0, 0) == 0) {
+ match = t;
+ debug(22, 5, "ttlSet: Matched '%s %d'\n",
+ match->pattern,
+ match->abs_ttl);
+ }
+ }
+ if (match) {
+ ttl = match->abs_ttl;
+ flags |= TTL_FORCE;
+ }
+ }
+
+ debug(22, 4, "ttlSet: [%c%c%c%c%c%c%c%c] %6.2lf days %s\n",
flags & TTL_EXPIRES ? 'E' : '.',
flags & TTL_SERVERDATE ? 'S' : '.',
@@ -290,7 +349,7 @@
flags & TTL_ABSOLUTE ? 'A' : '.',
flags & TTL_DEFAULT ? 'D' : '.',
+ flags & TTL_FORCE ? 'F' : '.',
(double) ttl / 86400, entry->url);
entry->expires = squid_curtime + ttl;
- entry->lastmod = last_modified > -1 ? last_modified : served_date;
}
--
Sincerely, srb@cuci.nl
Stephen R. van den Berg (AKA BuGless).
Father's Day Special at the local clinic -- Vasectomy!
Received on Thu Aug 01 1996 - 03:45:55 MDT
This archive was generated by hypermail pre-2.1.9 : Tue Dec 09 2003 - 16:32:43 MST