commit - cdfe5501a3d4265bab3c6f877f19f37a4b20b0da
commit + 1aaa1562179ea8be813abb49d6bdb197e0eb2c60
blob - a88ea5659f6ff7be69d846ef2825a6e32b5963a7
blob + aaa03170319bd65b941e76c4152b6d5720bdcd5a
--- lib/delta_cache.c
+++ lib/delta_cache.c
#define nitems(_a) (sizeof(_a) / sizeof((_a)[0]))
#endif
-#define GOT_DELTA_CACHE_MIN_BUCKETS 64
-#define GOT_DELTA_CACHE_MAX_BUCKETS 2048
-#define GOT_DELTA_CACHE_MAX_CHAIN 2
-#define GOT_DELTA_CACHE_MAX_DELTA_SIZE 1024
+#define GOT_DELTA_CACHE_MIN_BUCKETS 64
+#define GOT_DELTA_CACHE_MAX_BUCKETS 2048
+#define GOT_DELTA_CACHE_MAX_CHAIN 2
+#define GOT_DELTA_CACHE_MAX_DELTA_SIZE 1024
+#define GOT_DELTA_CACHE_MAX_FULLTEXT_SIZE 524288
+
struct got_cached_delta {
off_t offset;
uint8_t *data;
size_t len;
+ uint8_t *fulltext;
+ size_t fulltext_len;
};
struct got_delta_cache_head {
unsigned int totelem;
int cache_search;
int cache_hit;
+ int cache_hit_fulltext;
int cache_miss;
int cache_evict;
int cache_toolarge;
+ int cache_toolarge_fulltext;
int cache_maxtoolarge;
+ int cache_maxtoolarge_fulltext;
unsigned int flags;
#define GOT_DELTA_CACHE_F_NOMEM 0x01
SIPHASH_KEY key;
#ifdef GOT_DELTA_CACHE_DEBUG
fprintf(stderr, "%s: delta cache: %u elements, %d searches, %d hits, "
- "%d missed, %d evicted, %d too large (max %d)\n", getprogname(),
- cache->totelem, cache->cache_search, cache->cache_hit,
+ "%d fulltext hits, %d missed, %d evicted, %d too large (max %d), "
+ "%d too large fulltext (max %d)\n",
+ getprogname(), cache->totelem, cache->cache_search,
+ cache->cache_hit, cache->cache_hit_fulltext,
cache->cache_miss, cache->cache_evict, cache->cache_toolarge,
- cache->cache_maxtoolarge);
+ cache->cache_maxtoolarge,
+ cache->cache_toolarge_fulltext,
+ cache->cache_maxtoolarge_fulltext);
#endif
for (i = 0; i < cache->nbuckets; i++) {
struct got_delta_cache_head *head;
if (head->nchain >= nitems(head->entries)) {
delta = &head->entries[head->nchain - 1];
free(delta->data);
+ free(delta->fulltext);
memset(delta, 0, sizeof(*delta));
head->nchain--;
cache->totelem--;
delta->offset = delta_data_offset;
delta->data = delta_data;
delta->len = delta_len;
+ delta->fulltext = NULL;
+ delta->fulltext_len = 0;
head->nchain++;
cache->totelem++;
#endif
}
+const struct got_error *
+got_delta_cache_add_fulltext(struct got_delta_cache *cache,
+ off_t delta_data_offset, uint8_t *fulltext, size_t fulltext_len)
+{
+#ifdef GOT_NO_DELTA_CACHE
+ return got_error(GOT_ERR_NO_SPACE);
+#else
+ struct got_cached_delta *delta;
+ struct got_delta_cache_head *head;
+ uint64_t idx;
+ int i;
+
+ if (fulltext_len > GOT_DELTA_CACHE_MAX_FULLTEXT_SIZE) {
+ cache->cache_toolarge_fulltext++;
+ if (fulltext_len > cache->cache_maxtoolarge)
+ cache->cache_maxtoolarge_fulltext = fulltext_len;
+ return got_error(GOT_ERR_NO_SPACE);
+ }
+
+ idx = delta_cache_hash(cache, delta_data_offset) % cache->nbuckets;
+ head = &cache->buckets[idx];
+
+ for (i = 0; i < head->nchain; i++) {
+ delta = &head->entries[i];
+ if (delta->offset != delta_data_offset)
+ continue;
+ if (i > 0) {
+ struct got_cached_delta tmp;
+ memcpy(&tmp, &head->entries[0], sizeof(tmp));
+ memcpy(&head->entries[0], &head->entries[i],
+ sizeof(head->entries[0]));
+ memcpy(&head->entries[i], &tmp,
+ sizeof(head->entries[i]));
+ delta = &head->entries[0];
+ }
+ delta->fulltext = malloc(fulltext_len);
+ if (delta->fulltext == NULL)
+ return got_error_from_errno("malloc");
+ memcpy(delta->fulltext, fulltext, fulltext_len);
+ delta->fulltext_len = fulltext_len;
+ break;
+ }
+
+ return NULL;
+#endif
+}
+
void
got_delta_cache_get(uint8_t **delta_data, size_t *delta_len,
+ uint8_t **fulltext, size_t *fulltext_len,
struct got_delta_cache *cache, off_t delta_data_offset)
{
uint64_t idx;
cache->cache_search++;
*delta_data = NULL;
*delta_len = 0;
+ if (fulltext)
+ *fulltext = NULL;
+ if (fulltext_len)
+ *fulltext_len = 0;
for (i = 0; i < head->nchain; i++) {
delta = &head->entries[i];
if (delta->offset != delta_data_offset)
}
*delta_data = delta->data;
*delta_len = delta->len;
+ if (fulltext && fulltext_len &&
+ delta->fulltext && delta->fulltext_len) {
+ *fulltext = delta->fulltext;
+ *fulltext_len = delta->fulltext_len;
+ cache->cache_hit_fulltext++;
+ }
+
return;
}
blob - 7da86957201fd53ae69d5deb71b64d3bf6216599
blob + 03111a87e65fe8d7de030236276b4a0961ab8d05
--- lib/got_lib_delta_cache.h
+++ lib/got_lib_delta_cache.h
const struct got_error *got_delta_cache_add(struct got_delta_cache *, off_t,
uint8_t *, size_t);
-void got_delta_cache_get(uint8_t **, size_t *, struct got_delta_cache *, off_t);
+const struct got_error *got_delta_cache_add_fulltext(struct got_delta_cache *,
+ off_t , uint8_t *, size_t);
+void got_delta_cache_get(uint8_t **, size_t *, uint8_t **, size_t *,
+ struct got_delta_cache *, off_t);
blob - a9b71c809585eb6a270c9f49e602be196d76f3ad
blob + 4bef4fa0fe94d62da5c1003246d35e196a4e04dc
--- lib/pack.c
+++ lib/pack.c
if (pack->delta_cache) {
got_delta_cache_get(&delta_buf, &delta_len,
- pack->delta_cache, delta->data_offset);
+ NULL, NULL, pack->delta_cache,
+ delta->data_offset);
}
if (delta_buf == NULL) {
cached = 0;
const struct got_error *err = NULL;
struct got_delta *delta;
uint8_t *base_buf = NULL, *accum_buf = NULL;
- size_t base_bufsz = 0, accum_bufsz = 0, accum_size = 0, delta_len;
+ size_t base_bufsz = 0, accum_bufsz = 0, accum_size = 0;
/* We process small enough files entirely in memory for speed. */
const size_t max_bufsize = GOT_DELTA_RESULT_SIZE_CACHED_MAX;
uint64_t max_size = 0;
if (STAILQ_EMPTY(&deltas->entries))
return got_error(GOT_ERR_BAD_DELTA_CHAIN);
+
+ if (pack->delta_cache) {
+ uint8_t *delta_buf = NULL, *fulltext = NULL;
+ size_t delta_len, fulltext_len;
+
+ delta = STAILQ_LAST(&deltas->entries, got_delta, entry);
+ got_delta_cache_get(&delta_buf, &delta_len,
+ &fulltext, &fulltext_len,
+ pack->delta_cache, delta->data_offset);
+ if (fulltext) {
+ size_t w = fwrite(fulltext, 1, fulltext_len, outfile);
+ if (w != fulltext_len)
+ return got_ferror(outfile, GOT_ERR_IO);
+ *result_size = fulltext_len;
+ return NULL;
+ }
+ }
if (fseeko(base_file, 0L, SEEK_SET) == -1)
return got_error_from_errno("fseeko");
/* Deltas are ordered in ascending order. */
STAILQ_FOREACH(delta, &deltas->entries, entry) {
- uint8_t *delta_buf = NULL;
+ uint8_t *delta_buf = NULL, *fulltext = NULL;
+ size_t delta_len, fulltext_len;
uint64_t base_size, result_size = 0;
int cached = 1;
if (n == 0) {
if (pack->delta_cache) {
got_delta_cache_get(&delta_buf, &delta_len,
+ &fulltext, &fulltext_len,
pack->delta_cache, delta->data_offset);
}
if (delta_buf == NULL) {
max_size = base_size;
if (result_size > max_size)
max_size = result_size;
+ if (fulltext_len > max_size)
+ max_size = fulltext_len;
if (base_buf && max_size > max_bufsize) {
/* Switch from buffers to temporary files. */
}
if (base_buf) {
- err = got_delta_apply_in_mem(base_buf, base_bufsz,
- delta_buf, delta_len, accum_buf,
- &accum_size, max_size);
+ if (fulltext) {
+ memcpy(accum_buf, fulltext, fulltext_len);
+ accum_size = fulltext_len;
+ err = NULL;
+ } else {
+ err = got_delta_apply_in_mem(base_buf, base_bufsz,
+ delta_buf, delta_len, accum_buf,
+ &accum_size, max_size);
+ }
n++;
+ if (!cached)
+ free(delta_buf);
+ if (err)
+ goto done;
+ if (fulltext == NULL) {
+ err = got_delta_cache_add_fulltext(
+ pack->delta_cache, delta->data_offset,
+ accum_buf, accum_size);
+ if (err) {
+ if (err->code != GOT_ERR_NO_SPACE)
+ goto done;
+ err = NULL;
+ }
+ }
} else {
err = got_delta_apply(base_file, delta_buf,
delta_len,
/* Final delta application writes to output file. */
++n < deltas->nentries ? accum_file : outfile,
&accum_size);
+ if (!cached)
+ free(delta_buf);
+ if (err)
+ goto done;
}
- if (!cached)
- free(delta_buf);
- if (err)
- goto done;
if (n < deltas->nentries) {
/* Accumulated delta becomes the new base. */
const struct got_error *err = NULL;
struct got_delta *delta;
uint8_t *base_buf = NULL, *accum_buf = NULL;
- size_t base_bufsz = 0, accum_bufsz = 0, accum_size = 0, delta_len;
+ size_t base_bufsz = 0, accum_bufsz = 0, accum_size = 0;
uint64_t max_size = 0;
int n = 0;
if (STAILQ_EMPTY(&deltas->entries))
return got_error(GOT_ERR_BAD_DELTA_CHAIN);
+
+ if (pack->delta_cache) {
+ uint8_t *delta_buf = NULL, *fulltext = NULL;
+ size_t delta_len, fulltext_len;
+
+ delta = STAILQ_LAST(&deltas->entries, got_delta, entry);
+ got_delta_cache_get(&delta_buf, &delta_len,
+ &fulltext, &fulltext_len,
+ pack->delta_cache, delta->data_offset);
+ if (fulltext) {
+ *outbuf = malloc(fulltext_len);
+ if (*outbuf == NULL)
+ return got_error_from_errno("malloc");
+ memcpy(*outbuf, fulltext, fulltext_len);
+ *outlen = fulltext_len;
+ return NULL;
+ }
+ }
/* Deltas are ordered in ascending order. */
STAILQ_FOREACH(delta, &deltas->entries, entry) {
- uint8_t *delta_buf = NULL;
+ uint8_t *delta_buf = NULL, *fulltext = NULL;
+ size_t delta_len, fulltext_len = 0;
uint64_t base_size, result_size = 0;
int cached = 1;
if (n == 0) {
goto done;
}
+ if (pack->delta_cache) {
+ got_delta_cache_get(&delta_buf, &delta_len,
+ &fulltext, &fulltext_len,
+ pack->delta_cache, delta_data_offset);
+ }
+
if (delta->size > max_size)
max_size = delta->size;
+ if (delta->size > fulltext_len)
+ max_size = fulltext_len;
- if (pack->map) {
+ if (fulltext) {
+ base_buf = malloc(fulltext_len);
+ if (base_buf == NULL) {
+ err = got_error_from_errno("malloc");
+ goto done;
+ }
+ memcpy(base_buf, fulltext, fulltext_len);
+ base_bufsz = fulltext_len;
+ } else if (pack->map) {
size_t mapoff;
if (delta_data_offset > SIZE_MAX) {
if (err)
goto done;
n++;
+
+ if (pack->delta_cache && fulltext == NULL) {
+ err = got_delta_cache_add(pack->delta_cache,
+ delta_data_offset, NULL, 0);
+ if (err) {
+ if (err->code != GOT_ERR_NO_SPACE)
+ goto done;
+ err = NULL;
+ } else {
+ err = got_delta_cache_add_fulltext(
+ pack->delta_cache,
+ delta_data_offset,
+ fulltext, fulltext_len);
+ if (err && err->code != GOT_ERR_NO_SPACE)
+ goto done;
+ err = NULL;
+ }
+ }
continue;
}
if (pack->delta_cache) {
got_delta_cache_get(&delta_buf, &delta_len,
+ &fulltext, &fulltext_len,
pack->delta_cache, delta->data_offset);
}
if (delta_buf == NULL) {
max_size = base_size;
if (result_size > max_size)
max_size = result_size;
+ if (fulltext_len > max_size)
+ max_size = fulltext_len;
if (max_size > base_bufsz) {
uint8_t *p = realloc(base_buf, max_size);
accum_bufsz = max_size;
}
- err = got_delta_apply_in_mem(base_buf, base_bufsz,
- delta_buf, delta_len, accum_buf,
- &accum_size, max_size);
+ if (fulltext) {
+ memcpy(accum_buf, fulltext, fulltext_len);
+ accum_size = fulltext_len;
+ err = NULL;
+ } else {
+ err = got_delta_apply_in_mem(base_buf, base_bufsz,
+ delta_buf, delta_len, accum_buf,
+ &accum_size, max_size);
+ }
if (!cached)
free(delta_buf);
n++;
if (err)
goto done;
+ if (fulltext == NULL) {
+ err = got_delta_cache_add_fulltext(pack->delta_cache,
+ delta->data_offset, accum_buf, accum_size);
+ if (err) {
+ if (err->code != GOT_ERR_NO_SPACE)
+ goto done;
+ err = NULL;
+ }
+ }
+
if (n < deltas->nentries) {
/* Accumulated delta becomes the new base. */
uint8_t *tmp = accum_buf;
