commit - edfa7d7fafb6ce603f9463e5184d2586be36234e
commit + c59b334653ceb49c2009ed64fe3a16fa89e0e7f4
blob - fab42876b12d54898f6e2d271a7566f85276b99e
blob + fc8b2eef010a4a07760e360d7e9102d16791ddbd
--- lib/got_lib_object.h
+++ lib/got_lib_object.h
struct got_object_id id;
char *path_packfile; /* if packed */
+ int pack_idx; /* if packed */
off_t pack_offset; /* if packed */
struct got_delta_chain deltas; /* if deltified */
int refcnt; /* > 0 if open and/or cached */
blob - c36311dd363087ed4b79603a54bd1cf61fe22145
blob + 38a83477cfa367f9bd2c029371bbe1fbcf2f0598
--- lib/got_lib_privsep.h
+++ lib/got_lib_privsep.h
GOT_IMSG_PACKIDX,
GOT_IMSG_PACK,
GOT_IMSG_PACKED_OBJECT_REQUEST,
-
- /* Messages for transmitting deltas and associated delta streams: */
- GOT_IMSG_DELTA,
- GOT_IMSG_DELTA_STREAM,
};
/* Structure for GOT_IMSG_ERROR. */
int errno_code; /* in case code equals GOT_ERR_ERRNO */
};
-/* Structure for GOT_IMSG_DELTA data. */
-struct got_imsg_delta {
- /* These fields are the same as in struct got_delta. */
- off_t offset;
- size_t tslen;
- int type;
- size_t size;
- off_t data_offset;
- size_t delta_len;
-
- /*
- * Followed by one or more DELTA_STREAM messages until delta_len
- * bytes of delta stream have been transmitted.
- */
-};
-
-/* Structure for GOT_IMSG_DELTA_STREAM data. */
-struct got_imsg_delta_stream {
- /*
- * Empty since the following is implied:
- * Read additional delta stream data from imsg buffer.
- */
-};
-
/*
* Structure for GOT_IMSG_TREE_REQUEST, GOT_IMSG_COMMIT_REQUEST,
* and GOT_IMSG_OBJECT data.
*/
struct got_imsg_object {
+ uint8_t id[SHA1_DIGEST_LENGTH];
+
/* These fields are the same as in struct got_object. */
int type;
int flags;
size_t hdrlen;
size_t size;
-
off_t pack_offset;
- int ndeltas; /* this many GOT_IMSG_DELTA messages follow */
+ int pack_idx;
};
/* Structure for GOT_IMSG_COMMIT data. */
blob - adef4e516db7b524740a5a8192cc6d199d1d645a
blob + c10be9b25f2c42ae51559018bee66d6d981c67f9
--- lib/object_idcache.c
+++ lib/object_idcache.c
struct got_object_idcache_element *entry;
TAILQ_FOREACH(entry, &cache->entries, entry) {
- if (got_object_id_cmp(&entry->id, id) != 0)
+ if (memcmp(&entry->id.sha1, id->sha1, SHA1_DIGEST_LENGTH) != 0)
continue;
if (entry != TAILQ_FIRST(&cache->entries)) {
TAILQ_REMOVE(&cache->entries, entry, entry);
struct got_object_idcache_element *entry;
TAILQ_FOREACH(entry, &cache->entries, entry) {
- if (got_object_id_cmp(&entry->id, id) == 0)
+ if (memcmp(&entry->id.sha1, id->sha1, SHA1_DIGEST_LENGTH) == 0)
return 1;
}
blob - 00d5a158a66d76ebd4b9210a50d5bbabccef3c7b
blob + 3560e5c6fecdfae43a2c4fb617448f293a1db529
--- lib/pack.c
+++ lib/pack.c
static const struct got_error *
open_plain_object(struct got_object **obj, const char *path_packfile,
- struct got_object_id *id, uint8_t type, off_t offset, size_t size)
+ struct got_object_id *id, uint8_t type, off_t offset, size_t size, int idx)
{
*obj = calloc(1, sizeof(**obj));
if (*obj == NULL)
(*obj)->type = type;
(*obj)->flags = GOT_OBJ_FLAG_PACKED;
+ (*obj)->pack_idx = idx;
(*obj)->hdrlen = 0;
(*obj)->size = size;
if (id)
static const struct got_error *
open_delta_object(struct got_object **obj, struct got_packidx *packidx,
struct got_pack *pack, struct got_object_id *id, off_t offset,
- size_t tslen, int delta_type, size_t delta_size)
+ size_t tslen, int delta_type, size_t delta_size, int idx)
{
const struct got_error *err = NULL;
int resolved_type;
goto done;
}
(*obj)->flags |= GOT_OBJ_FLAG_PACKED;
+ (*obj)->pack_idx = idx;
SIMPLEQ_INIT(&(*obj)->deltas.entries);
(*obj)->flags |= GOT_OBJ_FLAG_DELTIFIED;
if (err)
goto done;
(*obj)->type = resolved_type;
-
done:
if (err) {
got_object_close(*obj);
case GOT_OBJ_TYPE_BLOB:
case GOT_OBJ_TYPE_TAG:
err = open_plain_object(obj, pack->path_packfile, id, type,
- offset + tslen, size);
+ offset + tslen, size, idx);
break;
case GOT_OBJ_TYPE_OFFSET_DELTA:
case GOT_OBJ_TYPE_REF_DELTA:
err = open_delta_object(obj, packidx, pack, id, offset,
- tslen, type, size);
+ tslen, type, size, idx);
break;
default:
err = got_error(GOT_ERR_OBJ_TYPE);
blob - b69b1fe537b7a8f35cb059168674e185ad81e1c7
blob + b101ac47cc0a29ad631f01913e378cc90cba9014
--- lib/privsep.c
+++ lib/privsep.c
err = flush_imsg(&ibuf);
imsg_clear(&ibuf);
return err;
-}
-
-static const struct got_error *
-send_delta(struct got_delta *delta, struct imsgbuf *ibuf)
-{
- struct got_imsg_delta idelta;
- size_t offset, remain;
-
- idelta.offset = delta->offset;
- idelta.tslen = delta->tslen;
- idelta.type = delta->type;
- idelta.size = delta->size;
- idelta.data_offset = delta->data_offset;
- idelta.delta_len = delta->delta_len;
-
- if (imsg_compose(ibuf, GOT_IMSG_DELTA, 0, 0, -1,
- &idelta, sizeof(idelta)) == -1)
- return got_error_from_errno();
-
- if (imsg_flush(ibuf) == -1)
- return got_error_from_errno();
-
- offset = 0;
- remain = delta->delta_len;
- while (remain > 0) {
- size_t n = MIN(MAX_IMSGSIZE - IMSG_HEADER_SIZE, remain);
-
- if (imsg_compose(ibuf, GOT_IMSG_DELTA_STREAM, 0, 0, -1,
- delta->delta_buf + offset, n) == -1)
- return got_error_from_errno();
-
- if (imsg_flush(ibuf) == -1)
- return got_error_from_errno();
-
- offset += n;
- remain -= n;
- }
-
- return NULL;
}
const struct got_error *
got_privsep_send_obj_req(struct imsgbuf *ibuf, int fd, struct got_object *obj)
{
- const struct got_error *err = NULL;
struct got_imsg_object iobj, *iobjp = NULL;
size_t iobj_size = 0;
int imsg_code = GOT_IMSG_OBJECT_REQUEST;
return got_error(GOT_ERR_OBJ_TYPE);
}
+ memcpy(iobj.id, obj->id.sha1, sizeof(iobj.id));
iobj.type = obj->type;
iobj.flags = obj->flags;
iobj.hdrlen = obj->hdrlen;
iobj.size = obj->size;
- iobj.ndeltas = obj->deltas.nentries;
- if (iobj.flags & GOT_OBJ_FLAG_PACKED)
+ if (iobj.flags & GOT_OBJ_FLAG_PACKED) {
iobj.pack_offset = obj->pack_offset;
+ iobj.pack_idx = obj->pack_idx;
+ }
iobjp = &iobj;
iobj_size = sizeof(iobj);
if (imsg_compose(ibuf, imsg_code, 0, 0, fd, iobjp, iobj_size) == -1)
return got_error_from_errno();
-
- err = flush_imsg(ibuf);
- if (err)
- return err;
- if (obj && obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
- struct got_delta *delta;
- SIMPLEQ_FOREACH(delta, &obj->deltas.entries, entry) {
- err = send_delta(delta, ibuf);
- if (err)
- break;
- }
- }
-
- return err;
+ return flush_imsg(ibuf);
}
const struct got_error *
const struct got_error *
got_privsep_send_obj(struct imsgbuf *ibuf, struct got_object *obj)
{
- const struct got_error *err = NULL;
struct got_imsg_object iobj;
- struct got_delta *delta;
+ memcpy(iobj.id, obj->id.sha1, sizeof(iobj.id));
iobj.type = obj->type;
iobj.flags = obj->flags;
iobj.hdrlen = obj->hdrlen;
iobj.size = obj->size;
- iobj.ndeltas = obj->deltas.nentries;
- if (iobj.flags & GOT_OBJ_FLAG_PACKED)
+ if (iobj.flags & GOT_OBJ_FLAG_PACKED) {
iobj.pack_offset = obj->pack_offset;
+ iobj.pack_idx = obj->pack_idx;
+ }
if (imsg_compose(ibuf, GOT_IMSG_OBJECT, 0, 0, -1, &iobj, sizeof(iobj))
== -1)
return got_error_from_errno();
- err = flush_imsg(ibuf);
- if (err)
- return err;
-
- SIMPLEQ_FOREACH(delta, &obj->deltas.entries, entry) {
- err = send_delta(delta, ibuf);
- if (err)
- break;
- }
-
- return err;
-}
-
-static const struct got_error *
-receive_delta(struct got_delta **delta, struct imsgbuf *ibuf)
-{
- const struct got_error *err = NULL;
- struct imsg imsg;
- struct got_imsg_delta idelta;
- uint8_t *delta_buf = NULL;
- const size_t min_datalen =
- MIN(sizeof(struct got_imsg_error), sizeof(struct got_imsg_delta));
- size_t datalen, offset, remain;
-
- err = got_privsep_recv_imsg(&imsg, ibuf, min_datalen);
- if (err)
- return err;
-
- datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
- if (imsg.hdr.type == GOT_IMSG_ERROR)
- return recv_imsg_error(&imsg, datalen);
-
- if (imsg.hdr.type != GOT_IMSG_DELTA)
- return got_error(GOT_ERR_PRIVSEP_MSG);
- if (datalen != sizeof(idelta))
- return got_error(GOT_ERR_PRIVSEP_LEN);
-
- memcpy(&idelta, imsg.data, sizeof(idelta));
- imsg_free(&imsg);
-
- switch (idelta.type) {
- case GOT_OBJ_TYPE_OFFSET_DELTA:
- case GOT_OBJ_TYPE_REF_DELTA:
- if (idelta.delta_len < GOT_DELTA_STREAM_LENGTH_MIN)
- return got_error(GOT_ERR_BAD_DELTA);
- break;
- default:
- if (idelta.delta_len != 0)
- return got_error(GOT_ERR_BAD_DELTA);
- break;
- }
-
- if (idelta.delta_len > 0) {
- delta_buf = calloc(1, idelta.delta_len);
- if (delta_buf == NULL)
- return got_error_from_errno();
-
- offset = 0;
- remain = idelta.delta_len;
- while (remain > 0) {
- size_t n = MIN(MAX_IMSGSIZE - IMSG_HEADER_SIZE, remain);
-
- err = got_privsep_recv_imsg(&imsg, ibuf, n);
- if (err)
- return err;
-
- if (imsg.hdr.type == GOT_IMSG_ERROR)
- return recv_imsg_error(&imsg, datalen);
-
- if (imsg.hdr.type == GOT_IMSG_STOP)
- break;
-
- if (imsg.hdr.type != GOT_IMSG_DELTA_STREAM)
- return got_error(GOT_ERR_PRIVSEP_MSG);
-
- datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
- if (datalen != n)
- return got_error(GOT_ERR_PRIVSEP_LEN);
-
- memcpy(delta_buf + offset, imsg.data, n);
- imsg_free(&imsg);
-
- offset += n;
- remain -= n;
- }
- }
-
- *delta = got_delta_open(idelta.offset, idelta.tslen, idelta.type,
- idelta.size, idelta.data_offset, delta_buf, idelta.delta_len);
- if (*delta == NULL) {
- err = got_error_from_errno();
- free(delta_buf);
- }
-
- return err;
+ return flush_imsg(ibuf);
}
const struct got_error *
const struct got_error *err = NULL;
struct got_imsg_object iobj;
size_t datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
- int i;
if (datalen != sizeof(iobj))
return got_error(GOT_ERR_PRIVSEP_LEN);
memcpy(&iobj, imsg->data, sizeof(iobj));
- if (iobj.ndeltas < 0 ||
- iobj.ndeltas > GOT_DELTA_CHAIN_RECURSION_MAX)
- return got_error(GOT_ERR_PRIVSEP_LEN);
- if (iobj.ndeltas > 0 &&
- (iobj.flags & GOT_OBJ_FLAG_DELTIFIED) == 0)
- return got_error(GOT_ERR_BAD_OBJ_DATA);
-
*obj = calloc(1, sizeof(**obj));
if (*obj == NULL)
return got_error_from_errno();
+ memcpy((*obj)->id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
(*obj)->type = iobj.type;
(*obj)->flags = iobj.flags;
(*obj)->hdrlen = iobj.hdrlen;
(*obj)->size = iobj.size;
- /* id and path_packfile might be copied in by caller */
- (*obj)->pack_offset = iobj.pack_offset;
- SIMPLEQ_INIT(&(*obj)->deltas.entries);
- for (i = 0; i < iobj.ndeltas; i++) {
- struct got_delta *delta;
- err = receive_delta(&delta, ibuf);
- if (err)
- break;
- (*obj)->deltas.nentries++;
- SIMPLEQ_INSERT_TAIL(&(*obj)->deltas.entries, delta,
- entry);
+ /* path_packfile is handled by caller */
+ if (iobj.flags & GOT_OBJ_FLAG_PACKED) {
+ (*obj)->pack_offset = iobj.pack_offset;
+ (*obj)->pack_idx = iobj.pack_idx;
}
return err;
blob - 682e940922ba11be438043238488405228f75f38
blob + ee42ef15b6041ff85391515838d089565cc8ebdb
--- lib/repository.c
+++ lib/repository.c
err = got_object_cache_add(&repo->commitcache, id, commit);
if (err)
return err;
-
commit->refcnt++;
#endif
return NULL;
blob - 688107f4d8b2a5ce11d51c5dab085e2ec0da4d97
blob + 1d11b5ccacd8a00511e57a311e6be23aa1eb2c64
--- libexec/got-read-pack/Makefile
+++ libexec/got-read-pack/Makefile
.PATH:${.CURDIR}/../../lib
PROG= got-read-pack
-SRCS= got-read-pack.c delta.c error.c inflate.c object_parse.c \
- opentemp.c pack.c path.c privsep.c sha1.c
+SRCS= got-read-pack.c delta.c error.c inflate.c object_cache.c \
+ object_idcache.c object_parse.c opentemp.c pack.c path.c \
+ privsep.c sha1.c
CPPFLAGS = -I${.CURDIR}/../../include -I${.CURDIR}/../../lib
LDADD = -lutil -lz
blob - c6634c689a2cfb7fe806ec3e4eedb5bc3cd70cd5
blob + 001b16f5d436b8d2a68d194127cb3fde8cac7d97
--- libexec/got-read-pack/got-read-pack.c
+++ libexec/got-read-pack/got-read-pack.c
#include "got_lib_delta.h"
#include "got_lib_inflate.h"
#include "got_lib_object.h"
+#include "got_lib_object_cache.h"
#include "got_lib_object_parse.h"
#include "got_lib_privsep.h"
#include "got_lib_pack.h"
static const struct got_error *
object_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
- struct got_packidx *packidx)
+ struct got_packidx *packidx, struct got_object_cache *objcache)
{
const struct got_error *err = NULL;
struct got_imsg_packed_object iobj;
err = got_packfile_open_object(&obj, pack, packidx, iobj.idx, NULL);
if (err)
return err;
+ obj->refcnt++;
+ err = got_object_cache_add(objcache, &obj->id, obj);
+ if (err)
+ goto done;
+ obj->refcnt++;
+
err = got_privsep_send_obj(ibuf, obj);
+done:
got_object_close(obj);
return err;
}
static const struct got_error *
+get_object(struct got_object **obj, struct imsg *imsg, struct imsgbuf *ibuf,
+ struct got_pack *pack, struct got_packidx *packidx,
+ struct got_object_cache *objcache, int type)
+{
+ const struct got_error *err = NULL;
+ struct got_object *iobj;
+
+ err = got_privsep_get_imsg_obj(&iobj, imsg, ibuf);
+ if (err)
+ return err;
+
+ if (iobj->type != type) {
+ err = got_error(GOT_ERR_OBJ_TYPE);
+ goto done;
+ }
+
+ if ((iobj->flags & GOT_OBJ_FLAG_PACKED) == 0)
+ return got_error(GOT_ERR_OBJ_NOT_PACKED);
+
+ *obj = got_object_cache_get(objcache, &iobj->id);
+ if (*obj == NULL) {
+ err = got_packfile_open_object(obj, pack, packidx,
+ iobj->pack_idx, &iobj->id);
+ if (err)
+ goto done;
+ }
+ (*obj)->refcnt++;
+done:
+ got_object_close(iobj);
+ return err;
+}
+
+static const struct got_error *
commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
- struct got_packidx *packidx)
+ struct got_packidx *packidx, struct got_object_cache *objcache)
{
const struct got_error *err = NULL;
struct got_object *obj = NULL;
uint8_t *buf;
size_t len;
- err = got_privsep_get_imsg_obj(&obj, imsg, ibuf);
+ err = get_object(&obj, imsg, ibuf, pack, packidx, objcache,
+ GOT_OBJ_TYPE_COMMIT);
if (err)
return err;
-
- if (obj->type != GOT_OBJ_TYPE_COMMIT)
- return got_error(GOT_ERR_OBJ_TYPE);
err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
if (err)
static const struct got_error *
tree_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
- struct got_packidx *packidx)
+ struct got_packidx *packidx, struct got_object_cache *objcache)
{
const struct got_error *err = NULL;
struct got_object *obj = NULL;
uint8_t *buf;
size_t len;
- err = got_privsep_get_imsg_obj(&obj, imsg, ibuf);
+ err = get_object(&obj, imsg, ibuf, pack, packidx, objcache,
+ GOT_OBJ_TYPE_TREE);
if (err)
return err;
- if (obj->type != GOT_OBJ_TYPE_TREE)
- return got_error(GOT_ERR_OBJ_TYPE);
-
err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
if (err)
return err;
static const struct got_error *
blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
- struct got_packidx *packidx)
+ struct got_packidx *packidx, struct got_object_cache *objcache)
{
const struct got_error *err = NULL;
struct got_object *obj = NULL;
memset(&imsg_outfd, 0, sizeof(imsg_outfd));
imsg_outfd.fd = -1;
- err = got_privsep_get_imsg_obj(&obj, imsg, ibuf);
+ err = get_object(&obj, imsg, ibuf, pack, packidx, objcache,
+ GOT_OBJ_TYPE_BLOB);
if (err)
return err;
- if (obj->type != GOT_OBJ_TYPE_BLOB)
- return got_error(GOT_ERR_OBJ_TYPE);
-
- if ((obj->flags & GOT_OBJ_FLAG_PACKED) == 0)
- return got_error(GOT_ERR_OBJ_NOT_PACKED);
-
err = got_privsep_recv_imsg(&imsg_outfd, ibuf, 0);
if (err)
return err;
else if (imsg_outfd.fd != -1)
close(imsg_outfd.fd);
imsg_free(&imsg_outfd);
-
+ if (obj)
+ got_object_close(obj);
if (err) {
if (err->code == GOT_ERR_PRIVSEP_PIPE)
err = NULL;
const struct got_error *err = NULL;
struct imsgbuf ibuf;
struct imsg imsg;
- struct got_packidx *packidx;
- struct got_pack *pack;
+ struct got_packidx *packidx = NULL;
+ struct got_pack *pack = NULL;
+ struct got_object_cache objcache;
//static int attached;
//while (!attached) sleep(1);
imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
+ err = got_object_cache_init(&objcache, GOT_OBJECT_CACHE_TYPE_OBJ);
+ if (err) {
+ err = got_error_from_errno();
+ got_privsep_send_error(&ibuf, err);
+ return 1;
+ }
+
/* revoke access to most system calls */
if (pledge("stdio recvfd", NULL) == -1) {
err = got_error_from_errno();
switch (imsg.hdr.type) {
case GOT_IMSG_PACKED_OBJECT_REQUEST:
- err = object_request(&imsg, &ibuf, pack, packidx);
+ err = object_request(&imsg, &ibuf, pack, packidx,
+ &objcache);
break;
case GOT_IMSG_COMMIT_REQUEST:
- err = commit_request(&imsg, &ibuf, pack, packidx);
+ err = commit_request(&imsg, &ibuf, pack, packidx,
+ &objcache);
break;
case GOT_IMSG_TREE_REQUEST:
- err = tree_request(&imsg, &ibuf, pack, packidx);
+ err = tree_request(&imsg, &ibuf, pack, packidx,
+ &objcache);
break;
case GOT_IMSG_BLOB_REQUEST:
- err = blob_request(&imsg, &ibuf, pack, packidx);
+ err = blob_request(&imsg, &ibuf, pack, packidx,
+ &objcache);
break;
default:
err = got_error(GOT_ERR_PRIVSEP_MSG);
}
}
- got_pack_close(pack);
+ if (packidx)
+ got_packidx_close(packidx);
+ if (pack)
+ got_pack_close(pack);
imsg_clear(&ibuf);
if (err)
fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
