commit - 2d28509da5e9083d173044b04757f67a0fa66c28
commit + f7a026c46d9d66483e01a412330978fb70c4427e
blob - 80dbeec473acf23e7d088b05941b2f804b5dba45
blob + a671e0d9a139fda4a7595ab20adc7b3d85006678
--- lib/delta_cache.c
+++ lib/delta_cache.c
@@ -39,15 +39,19 @@
 #define nitems(_a) (sizeof(_a) / sizeof((_a)[0]))
 #endif
 
-#define GOT_DELTA_CACHE_MIN_BUCKETS	64
-#define GOT_DELTA_CACHE_MAX_BUCKETS	2048
-#define GOT_DELTA_CACHE_MAX_CHAIN	2
-#define GOT_DELTA_CACHE_MAX_DELTA_SIZE	1024
+#define GOT_DELTA_CACHE_MIN_BUCKETS		64
+#define GOT_DELTA_CACHE_MAX_BUCKETS		2048
+#define GOT_DELTA_CACHE_MAX_CHAIN		2
+#define GOT_DELTA_CACHE_MAX_DELTA_SIZE		1024
+#define GOT_DELTA_CACHE_MAX_FULLTEXT_SIZE	524288
 
+
 struct got_cached_delta {
 	off_t offset;
 	uint8_t *data;
 	size_t len;
+	uint8_t *fulltext;
+	size_t fulltext_len;
 };
 
 struct got_delta_cache_head {
@@ -61,10 +65,13 @@ struct got_delta_cache {
 	unsigned int totelem;
 	int cache_search;
 	int cache_hit;
+	int cache_hit_fulltext;
 	int cache_miss;
 	int cache_evict;
 	int cache_toolarge;
+	int cache_toolarge_fulltext;
 	int cache_maxtoolarge;
+	int cache_maxtoolarge_fulltext;
 	unsigned int flags;
 #define GOT_DELTA_CACHE_F_NOMEM	0x01
 	SIPHASH_KEY key;
@@ -104,10 +111,14 @@ got_delta_cache_free(struct got_delta_cache *cache)
 
 #ifdef GOT_DELTA_CACHE_DEBUG
 	fprintf(stderr, "%s: delta cache: %u elements, %d searches, %d hits, "
-	    "%d missed, %d evicted, %d too large (max %d)\n", getprogname(),
-	    cache->totelem, cache->cache_search, cache->cache_hit,
+	    "%d fulltext hits, %d missed, %d evicted, %d too large (max %d), "
+	    "%d too large fulltext (max %d)\n",
+	    getprogname(), cache->totelem, cache->cache_search,
+	    cache->cache_hit, cache->cache_hit_fulltext,
 	    cache->cache_miss, cache->cache_evict, cache->cache_toolarge,
-	    cache->cache_maxtoolarge);
+	    cache->cache_maxtoolarge,
+	    cache->cache_toolarge_fulltext,
+	    cache->cache_maxtoolarge_fulltext);
 #endif
 	for (i = 0; i < cache->nbuckets; i++) {
 		struct got_delta_cache_head *head;
@@ -221,6 +232,7 @@ got_delta_cache_add(struct got_delta_cache *cache,
 	if (head->nchain >= nitems(head->entries)) {
 		delta = &head->entries[head->nchain - 1];
 		free(delta->data);
+		free(delta->fulltext);
 		memset(delta, 0, sizeof(*delta));
 		head->nchain--;
 		cache->totelem--;
@@ -231,6 +243,8 @@ got_delta_cache_add(struct got_delta_cache *cache,
 	delta->offset = delta_data_offset;
 	delta->data = delta_data;
 	delta->len = delta_len;
+	delta->fulltext = NULL;
+	delta->fulltext_len = 0;
 	head->nchain++;
 	cache->totelem++;
 
@@ -238,8 +252,56 @@ got_delta_cache_add(struct got_delta_cache *cache,
 #endif
 }
 
+const struct got_error *
+got_delta_cache_add_fulltext(struct got_delta_cache *cache,
+    off_t delta_data_offset, uint8_t *fulltext, size_t fulltext_len)
+{
+#ifdef GOT_NO_DELTA_CACHE
+	return got_error(GOT_ERR_NO_SPACE);
+#else
+	struct got_cached_delta *delta;
+	struct got_delta_cache_head *head;
+	uint64_t idx;
+	int i;
+
+	if (fulltext_len > GOT_DELTA_CACHE_MAX_FULLTEXT_SIZE) {
+		cache->cache_toolarge_fulltext++;
+		if (fulltext_len > cache->cache_maxtoolarge)
+			cache->cache_maxtoolarge_fulltext = fulltext_len;
+		return got_error(GOT_ERR_NO_SPACE);
+	}
+
+	idx = delta_cache_hash(cache, delta_data_offset) % cache->nbuckets;
+	head = &cache->buckets[idx];
+
+	for (i = 0; i < head->nchain; i++) {
+		delta = &head->entries[i];
+		if (delta->offset != delta_data_offset)
+			continue;
+		if (i > 0) {
+			struct got_cached_delta tmp;
+			memcpy(&tmp, &head->entries[0], sizeof(tmp));
+			memcpy(&head->entries[0], &head->entries[i],
+			    sizeof(head->entries[0]));
+			memcpy(&head->entries[i], &tmp,
+			    sizeof(head->entries[i]));
+			delta = &head->entries[0];
+		}
+		delta->fulltext = malloc(fulltext_len);
+		if (delta->fulltext == NULL)
+			return got_error_from_errno("malloc");
+		memcpy(delta->fulltext, fulltext, fulltext_len);
+		delta->fulltext_len = fulltext_len;
+		break;
+	}
+
+	return NULL;
+#endif
+}
+
 void
 got_delta_cache_get(uint8_t **delta_data, size_t *delta_len,
+    uint8_t **fulltext, size_t *fulltext_len,
     struct got_delta_cache *cache, off_t delta_data_offset)
 {
 	uint64_t idx;
@@ -253,6 +315,10 @@ got_delta_cache_get(uint8_t **delta_data, size_t *delt
 	cache->cache_search++;
 	*delta_data = NULL;
 	*delta_len = 0;
+	if (fulltext)
+		*fulltext = NULL;
+	if (fulltext_len)
+		*fulltext_len = 0;
 	for (i = 0; i < head->nchain; i++) {
 		delta = &head->entries[i];
 		if (delta->offset != delta_data_offset)
@@ -269,6 +335,13 @@ got_delta_cache_get(uint8_t **delta_data, size_t *delt
 		}
 		*delta_data = delta->data;
 		*delta_len = delta->len;
+		if (fulltext && fulltext_len &&
+		    delta->fulltext && delta->fulltext_len) {
+			*fulltext = delta->fulltext;
+			*fulltext_len = delta->fulltext_len;
+			cache->cache_hit_fulltext++;
+		}
+
 		return;
 	}
 
blob - 7da86957201fd53ae69d5deb71b64d3bf6216599
blob + 03111a87e65fe8d7de030236276b4a0961ab8d05
--- lib/got_lib_delta_cache.h
+++ lib/got_lib_delta_cache.h
@@ -21,4 +21,7 @@ void got_delta_cache_free(struct got_delta_cache *);
 
 const struct got_error *got_delta_cache_add(struct got_delta_cache *, off_t,
     uint8_t *, size_t);
-void got_delta_cache_get(uint8_t **, size_t *, struct got_delta_cache *, off_t);
+const struct got_error *got_delta_cache_add_fulltext(struct got_delta_cache *,
+    off_t , uint8_t *, size_t);
+void got_delta_cache_get(uint8_t **, size_t *, uint8_t **, size_t *,
+    struct got_delta_cache *, off_t);
blob - 2b3827d2bd6033b3a694f474b6d68ae211330bae
blob + 07f931d1c925ce8921b8afdc3c488c8fa5a66b64
--- lib/pack.c
+++ lib/pack.c
@@ -1313,7 +1313,8 @@ got_pack_get_delta_chain_max_size(uint64_t *max_size,
 
 			if (pack->delta_cache) {
 				got_delta_cache_get(&delta_buf, &delta_len,
-				    pack->delta_cache, delta->data_offset);
+				    NULL, NULL, pack->delta_cache,
+				    delta->data_offset);
 			}
 			if (delta_buf == NULL) {
 				cached = 0;
@@ -1367,7 +1368,7 @@ got_pack_dump_delta_chain_to_file(size_t *result_size,
 	const struct got_error *err = NULL;
 	struct got_delta *delta;
 	uint8_t *base_buf = NULL, *accum_buf = NULL;
-	size_t base_bufsz = 0, accum_bufsz = 0, accum_size = 0, delta_len;
+	size_t base_bufsz = 0, accum_bufsz = 0, accum_size = 0;
 	/* We process small enough files entirely in memory for speed. */
 	const size_t max_bufsize = GOT_DELTA_RESULT_SIZE_CACHED_MAX;
 	uint64_t max_size = 0;
@@ -1377,6 +1378,23 @@ got_pack_dump_delta_chain_to_file(size_t *result_size,
 
 	if (STAILQ_EMPTY(&deltas->entries))
 		return got_error(GOT_ERR_BAD_DELTA_CHAIN);
+
+	if (pack->delta_cache) {
+		uint8_t *delta_buf = NULL, *fulltext = NULL;
+		size_t delta_len, fulltext_len;
+
+		delta = STAILQ_LAST(&deltas->entries, got_delta, entry);
+		got_delta_cache_get(&delta_buf, &delta_len,
+		    &fulltext, &fulltext_len,
+		    pack->delta_cache, delta->data_offset);
+		if (fulltext) {
+			size_t w = fwrite(fulltext, 1, fulltext_len, outfile);
+			if (w != fulltext_len)
+				return got_ferror(outfile, GOT_ERR_IO);
+			*result_size = fulltext_len;
+			return NULL;
+		}
+	}
 
 	if (fseeko(base_file, 0L, SEEK_SET) == -1)
 		return got_error_from_errno("fseeko");
@@ -1385,7 +1403,8 @@ got_pack_dump_delta_chain_to_file(size_t *result_size,
 
 	/* Deltas are ordered in ascending order. */
 	STAILQ_FOREACH(delta, &deltas->entries, entry) {
-		uint8_t *delta_buf = NULL;
+		uint8_t *delta_buf = NULL, *fulltext = NULL;
+		size_t delta_len, fulltext_len;
 		uint64_t base_size, result_size = 0;
 		int cached = 1;
 		if (n == 0) {
@@ -1472,6 +1491,7 @@ got_pack_dump_delta_chain_to_file(size_t *result_size,
 
 		if (pack->delta_cache) {
 			got_delta_cache_get(&delta_buf, &delta_len,
+			    &fulltext, &fulltext_len,
 			    pack->delta_cache, delta->data_offset);
 		}
 		if (delta_buf == NULL) {
@@ -1503,6 +1523,8 @@ got_pack_dump_delta_chain_to_file(size_t *result_size,
 			max_size = base_size;
 		if (result_size > max_size)
 			max_size = result_size;
+		if (fulltext_len > max_size)
+			max_size = fulltext_len;
 
 		if (base_buf && max_size > max_bufsize) {
 			/* Switch from buffers to temporary files. */
@@ -1545,21 +1567,41 @@ got_pack_dump_delta_chain_to_file(size_t *result_size,
 		}
 
 		if (base_buf) {
-			err = got_delta_apply_in_mem(base_buf, base_bufsz,
-			    delta_buf, delta_len, accum_buf,
-			    &accum_size, max_size);
+			if (fulltext) {
+				memcpy(accum_buf, fulltext, fulltext_len);
+				accum_size = fulltext_len;
+				err = NULL;
+			} else {
+				err = got_delta_apply_in_mem(base_buf, base_bufsz,
+				    delta_buf, delta_len, accum_buf,
+				    &accum_size, max_size);
+			}
 			n++;
+			if (!cached)
+				free(delta_buf);
+			if (err)
+				goto done;
+			if (fulltext == NULL) {
+				err = got_delta_cache_add_fulltext(
+				    pack->delta_cache, delta->data_offset,
+				    accum_buf, accum_size);
+				if (err) {
+					if (err->code != GOT_ERR_NO_SPACE)
+						goto done;
+					err = NULL;
+				}
+			}
 		} else {
 			err = got_delta_apply(base_file, delta_buf,
 			    delta_len,
 			    /* Final delta application writes to output file. */
 			    ++n < deltas->nentries ? accum_file : outfile,
 			    &accum_size);
+			if (!cached)
+				free(delta_buf);
+			if (err)
+				goto done;
 		}
-		if (!cached)
-			free(delta_buf);
-		if (err)
-			goto done;
 
 		if (n < deltas->nentries) {
 			/* Accumulated delta becomes the new base. */
@@ -1601,7 +1643,7 @@ got_pack_dump_delta_chain_to_mem(uint8_t **outbuf, siz
 	const struct got_error *err = NULL;
 	struct got_delta *delta;
 	uint8_t *base_buf = NULL, *accum_buf = NULL;
-	size_t base_bufsz = 0, accum_bufsz = 0, accum_size = 0, delta_len;
+	size_t base_bufsz = 0, accum_bufsz = 0, accum_size = 0;
 	uint64_t max_size = 0;
 	int n = 0;
 
@@ -1610,10 +1652,29 @@ got_pack_dump_delta_chain_to_mem(uint8_t **outbuf, siz
 
 	if (STAILQ_EMPTY(&deltas->entries))
 		return got_error(GOT_ERR_BAD_DELTA_CHAIN);
+
+	if (pack->delta_cache) {
+		uint8_t *delta_buf = NULL, *fulltext = NULL;
+		size_t delta_len, fulltext_len;
+
+		delta = STAILQ_LAST(&deltas->entries, got_delta, entry);
+		got_delta_cache_get(&delta_buf, &delta_len,
+		    &fulltext, &fulltext_len,
+		    pack->delta_cache, delta->data_offset);
+		if (fulltext) {
+			*outbuf = malloc(fulltext_len);
+			if (*outbuf == NULL)
+				return got_error_from_errno("malloc");
+			memcpy(*outbuf, fulltext, fulltext_len);
+			*outlen = fulltext_len;
+			return NULL;
+		}
+	}
 
 	/* Deltas are ordered in ascending order. */
 	STAILQ_FOREACH(delta, &deltas->entries, entry) {
-		uint8_t *delta_buf = NULL;
+		uint8_t *delta_buf = NULL, *fulltext = NULL;
+		size_t delta_len, fulltext_len = 0;
 		uint64_t base_size, result_size = 0;
 		int cached = 1;
 		if (n == 0) {
@@ -1634,10 +1695,26 @@ got_pack_dump_delta_chain_to_mem(uint8_t **outbuf, siz
 				goto done;
 			}
 
+			if (pack->delta_cache) {
+				got_delta_cache_get(&delta_buf, &delta_len,
+				    &fulltext, &fulltext_len,
+				    pack->delta_cache, delta_data_offset);
+			}
+
 			if (delta->size > max_size)
 				max_size = delta->size;
+			if (delta->size > fulltext_len)
+				max_size = fulltext_len;
 
-			if (pack->map) {
+			if (fulltext) {
+				base_buf = malloc(fulltext_len);
+				if (base_buf == NULL) {
+					err = got_error_from_errno("malloc");
+					goto done;
+				}
+				memcpy(base_buf, fulltext, fulltext_len);
+				base_bufsz = fulltext_len;
+			} else if (pack->map) {
 				size_t mapoff;
 
 				if (delta_data_offset > SIZE_MAX) {
@@ -1664,11 +1741,30 @@ got_pack_dump_delta_chain_to_mem(uint8_t **outbuf, siz
 			if (err)
 				goto done;
 			n++;
+
+			if (pack->delta_cache && fulltext == NULL) {
+				err = got_delta_cache_add(pack->delta_cache,
+				    delta_data_offset, NULL, 0);
+				if (err) {
+					if (err->code != GOT_ERR_NO_SPACE)
+						goto done;
+					err = NULL;
+				} else {
+					err = got_delta_cache_add_fulltext(
+					    pack->delta_cache,
+					    delta_data_offset,
+					    fulltext, fulltext_len);
+					if (err && err->code != GOT_ERR_NO_SPACE)
+						goto done;
+					err = NULL;
+				}
+			}
 			continue;
 		}
 
 		if (pack->delta_cache) {
 			got_delta_cache_get(&delta_buf, &delta_len,
+			    &fulltext, &fulltext_len,
 			    pack->delta_cache, delta->data_offset);
 		}
 		if (delta_buf == NULL) {
@@ -1700,6 +1796,8 @@ got_pack_dump_delta_chain_to_mem(uint8_t **outbuf, siz
 			max_size = base_size;
 		if (result_size > max_size)
 			max_size = result_size;
+		if (fulltext_len > max_size)
+			max_size = fulltext_len;
 
 		if (max_size > base_bufsz) {
 			uint8_t *p = realloc(base_buf, max_size);
@@ -1725,15 +1823,31 @@ got_pack_dump_delta_chain_to_mem(uint8_t **outbuf, siz
 			accum_bufsz = max_size;
 		}
 
-		err = got_delta_apply_in_mem(base_buf, base_bufsz,
-		    delta_buf, delta_len, accum_buf,
-		    &accum_size, max_size);
+		if (fulltext) {
+			memcpy(accum_buf, fulltext, fulltext_len);
+			accum_size = fulltext_len;
+			err = NULL;
+		} else {
+			err = got_delta_apply_in_mem(base_buf, base_bufsz,
+			    delta_buf, delta_len, accum_buf,
+			    &accum_size, max_size);
+		}
 		if (!cached)
 			free(delta_buf);
 		n++;
 		if (err)
 			goto done;
 
+		if (fulltext == NULL) {
+			err = got_delta_cache_add_fulltext(pack->delta_cache,
+			    delta->data_offset, accum_buf, accum_size);
+			if (err) {
+				if (err->code != GOT_ERR_NO_SPACE)
+					goto done;
+				err = NULL;
+			}
+		}
+
 		if (n < deltas->nentries) {
 			/* Accumulated delta becomes the new base. */
 			uint8_t *tmp = accum_buf;