/* * This file is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License, version 2, * as published by the Free Software Foundation. * * In addition to the permissions in the GNU General Public License, * the authors give you unlimited permission to link the compiled * version of this file into combinations with other programs, * and to distribute those combinations without any restriction * coming from the use of this file. (The General Public License * restrictions do apply in other respects; for example, they cover * modification of the file, and distribution when not linked into * a combined executable.) * * This file is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; see the file COPYING. If not, write to * the Free Software Foundation, 51 Franklin Street, Fifth Floor, * Boston, MA 02110-1301, USA. */ #include "common.h" #include "git2/zlib.h" #include "git2/object.h" #include "fileops.h" #include "hash.h" #include "odb.h" #include "delta-apply.h" #include "git2/odb_backend.h" typedef struct { /* object header data */ git_otype type; /* object type */ size_t size; /* object size */ } obj_hdr; typedef struct loose_backend { git_odb_backend parent; int object_zlib_level; /** loose object zlib compression level. */ int fsync_object_files; /** loose object file fsync flag. */ char *objects_dir; } loose_backend; /*********************************************************** * * MISCELANEOUS HELPER FUNCTIONS * ***********************************************************/ static int make_temp_file(git_file *fd, char *tmp, size_t n, char *file) { char *template = "/tmp_obj_XXXXXX"; size_t tmplen = strlen(template); int dirlen; if ((dirlen = git__dirname_r(tmp, n, file)) < 0) return GIT_ERROR; if ((dirlen + tmplen) >= n) return GIT_ERROR; strcpy(tmp + dirlen, (dirlen) ? template : template + 1); *fd = gitfo_mkstemp(tmp); if (*fd < 0 && dirlen) { /* create directory if it doesn't exist */ tmp[dirlen] = '\0'; if ((gitfo_exists(tmp) < 0) && gitfo_mkdir(tmp, 0755)) return GIT_ERROR; /* try again */ strcpy(tmp + dirlen, template); *fd = gitfo_mkstemp(tmp); } if (*fd < 0) return GIT_ERROR; return GIT_SUCCESS; } static size_t object_file_name(char *name, size_t n, char *dir, const git_oid *id) { size_t len = strlen(dir); /* check length: 43 = 40 hex sha1 chars + 2 * '/' + '\0' */ if (len+43 > n) return len+43; /* the object dir: eg $GIT_DIR/objects */ strcpy(name, dir); if (name[len-1] != '/') name[len++] = '/'; /* loose object filename: aa/aaa... (41 bytes) */ git_oid_pathfmt(&name[len], id); name[len+41] = '\0'; return 0; } static size_t get_binary_object_header(obj_hdr *hdr, gitfo_buf *obj) { unsigned char c; unsigned char *data = obj->data; size_t shift, size, used = 0; if (obj->len == 0) return 0; c = data[used++]; hdr->type = (c >> 4) & 7; size = c & 15; shift = 4; while (c & 0x80) { if (obj->len <= used) return 0; if (sizeof(size_t) * 8 <= shift) return 0; c = data[used++]; size += (c & 0x7f) << shift; shift += 7; } hdr->size = size; return used; } static size_t get_object_header(obj_hdr *hdr, unsigned char *data) { char c, typename[10]; size_t size, used = 0; /* * type name string followed by space. */ while ((c = data[used]) != ' ') { typename[used++] = c; if (used >= sizeof(typename)) return 0; } typename[used] = 0; if (used == 0) return 0; hdr->type = git_object_string2type(typename); used++; /* consume the space */ /* * length follows immediately in decimal (without * leading zeros). */ size = data[used++] - '0'; if (size > 9) return 0; if (size) { while ((c = data[used]) != '\0') { size_t d = c - '0'; if (d > 9) break; used++; size = size * 10 + d; } } hdr->size = size; /* * the length must be followed by a zero byte */ if (data[used++] != '\0') return 0; return used; } /*********************************************************** * * ZLIB RELATED FUNCTIONS * ***********************************************************/ static void init_stream(z_stream *s, void *out, size_t len) { memset(s, 0, sizeof(*s)); s->next_out = out; s->avail_out = len; } static void set_stream_input(z_stream *s, void *in, size_t len) { s->next_in = in; s->avail_in = len; } static void set_stream_output(z_stream *s, void *out, size_t len) { s->next_out = out; s->avail_out = len; } static int start_inflate(z_stream *s, gitfo_buf *obj, void *out, size_t len) { int status; init_stream(s, out, len); set_stream_input(s, obj->data, obj->len); if ((status = inflateInit(s)) < Z_OK) return status; return inflate(s, 0); } static int finish_inflate(z_stream *s) { int status = Z_OK; while (status == Z_OK) status = inflate(s, Z_FINISH); inflateEnd(s); if ((status != Z_STREAM_END) || (s->avail_in != 0)) return GIT_ERROR; return GIT_SUCCESS; } static int deflate_buf(z_stream *s, void *in, size_t len, int flush) { int status = Z_OK; set_stream_input(s, in, len); while (status == Z_OK) { status = deflate(s, flush); if (s->avail_in == 0) break; } return status; } static int deflate_obj(gitfo_buf *buf, char *hdr, int hdrlen, git_rawobj *obj, int level) { z_stream zs; int status; size_t size; assert(buf && !buf->data && hdr && obj); assert(level == Z_DEFAULT_COMPRESSION || (level >= 0 && level <= 9)); buf->data = NULL; buf->len = 0; init_stream(&zs, NULL, 0); if (deflateInit(&zs, level) < Z_OK) return GIT_ERROR; size = deflateBound(&zs, hdrlen + obj->len); if ((buf->data = git__malloc(size)) == NULL) { deflateEnd(&zs); return GIT_ERROR; } set_stream_output(&zs, buf->data, size); /* compress the header */ status = deflate_buf(&zs, hdr, hdrlen, Z_NO_FLUSH); /* if header compressed OK, compress the object */ if (status == Z_OK) status = deflate_buf(&zs, obj->data, obj->len, Z_FINISH); if (status != Z_STREAM_END) { deflateEnd(&zs); free(buf->data); buf->data = NULL; return GIT_ERROR; } buf->len = zs.total_out; deflateEnd(&zs); return GIT_SUCCESS; } static int is_zlib_compressed_data(unsigned char *data) { unsigned int w; w = ((unsigned int)(data[0]) << 8) + data[1]; return data[0] == 0x78 && !(w % 31); } static void *inflate_tail(z_stream *s, void *hb, size_t used, obj_hdr *hdr) { unsigned char *buf, *head = hb; size_t tail; /* * allocate a buffer to hold the inflated data and copy the * initial sequence of inflated data from the tail of the * head buffer, if any. */ if ((buf = git__malloc(hdr->size + 1)) == NULL) { inflateEnd(s); return NULL; } tail = s->total_out - used; if (used > 0 && tail > 0) { if (tail > hdr->size) tail = hdr->size; memcpy(buf, head + used, tail); } used = tail; /* * inflate the remainder of the object data, if any */ if (hdr->size < used) inflateEnd(s); else { set_stream_output(s, buf + used, hdr->size - used); if (finish_inflate(s)) { free(buf); return NULL; } } return buf; } /* * At one point, there was a loose object format that was intended to * mimic the format used in pack-files. This was to allow easy copying * of loose object data into packs. This format is no longer used, but * we must still read it. */ static int inflate_packlike_loose_disk_obj(git_rawobj *out, gitfo_buf *obj) { unsigned char *in, *buf; obj_hdr hdr; size_t len, used; /* * read the object header, which is an (uncompressed) * binary encoding of the object type and size. */ if ((used = get_binary_object_header(&hdr, obj)) == 0) return GIT_ERROR; if (!git_object_typeisloose(hdr.type)) return GIT_ERROR; /* * allocate a buffer and inflate the data into it */ buf = git__malloc(hdr.size + 1); if (!buf) return GIT_ERROR; in = ((unsigned char *)obj->data) + used; len = obj->len - used; if (git_odb__inflate_buffer(in, len, buf, hdr.size)) { free(buf); return GIT_ERROR; } buf[hdr.size] = '\0'; out->data = buf; out->len = hdr.size; out->type = hdr.type; return GIT_SUCCESS; } static int inflate_disk_obj(git_rawobj *out, gitfo_buf *obj) { unsigned char head[64], *buf; z_stream zs; int z_status; obj_hdr hdr; size_t used; /* * check for a pack-like loose object */ if (!is_zlib_compressed_data(obj->data)) return inflate_packlike_loose_disk_obj(out, obj); /* * inflate the initial part of the io buffer in order * to parse the object header (type and size). */ if ((z_status = start_inflate(&zs, obj, head, sizeof(head))) < Z_OK) return GIT_ERROR; if ((used = get_object_header(&hdr, head)) == 0) return GIT_ERROR; if (!git_object_typeisloose(hdr.type)) return GIT_ERROR; /* * allocate a buffer and inflate the object data into it * (including the initial sequence in the head buffer). */ if ((buf = inflate_tail(&zs, head, used, &hdr)) == NULL) return GIT_ERROR; buf[hdr.size] = '\0'; out->data = buf; out->len = hdr.size; out->type = hdr.type; return GIT_SUCCESS; } /*********************************************************** * * ODB OBJECT READING & WRITING * * Backend for the public API; read headers and full objects * from the ODB. Write raw data to the ODB. * ***********************************************************/ static int read_loose(git_rawobj *out, const char *loc) { int error; gitfo_buf obj = GITFO_BUF_INIT; assert(out && loc); out->data = NULL; out->len = 0; out->type = GIT_OBJ_BAD; if (gitfo_read_file(&obj, loc) < 0) return GIT_ENOTFOUND; error = inflate_disk_obj(out, &obj); gitfo_free_buf(&obj); return error; } static int read_header_loose(git_rawobj *out, const char *loc) { int error = GIT_SUCCESS, z_return = Z_ERRNO, read_bytes; git_file fd; z_stream zs; obj_hdr header_obj; unsigned char raw_buffer[16], inflated_buffer[64]; assert(out && loc); out->data = NULL; if ((fd = gitfo_open(loc, O_RDONLY)) < 0) return GIT_ENOTFOUND; init_stream(&zs, inflated_buffer, sizeof(inflated_buffer)); if (inflateInit(&zs) < Z_OK) { error = GIT_EZLIB; goto cleanup; } do { if ((read_bytes = read(fd, raw_buffer, sizeof(raw_buffer))) > 0) { set_stream_input(&zs, raw_buffer, read_bytes); z_return = inflate(&zs, 0); } } while (z_return == Z_OK); if ((z_return != Z_STREAM_END && z_return != Z_BUF_ERROR) || get_object_header(&header_obj, inflated_buffer) == 0 || git_object_typeisloose(header_obj.type) == 0) { error = GIT_EOBJCORRUPTED; goto cleanup; } out->len = header_obj.size; out->type = header_obj.type; cleanup: finish_inflate(&zs); gitfo_close(fd); return error; } static int write_obj(gitfo_buf *buf, git_oid *id, loose_backend *backend) { char file[GIT_PATH_MAX]; char temp[GIT_PATH_MAX]; git_file fd; if (object_file_name(file, sizeof(file), backend->objects_dir, id)) return GIT_EOSERR; if (make_temp_file(&fd, temp, sizeof(temp), file) < 0) return GIT_EOSERR; if (gitfo_write(fd, buf->data, buf->len) < 0) { gitfo_close(fd); gitfo_unlink(temp); return GIT_EOSERR; } if (backend->fsync_object_files) gitfo_fsync(fd); gitfo_close(fd); gitfo_chmod(temp, 0444); if (gitfo_mv(temp, file) < 0) { gitfo_unlink(temp); return GIT_EOSERR; } return GIT_SUCCESS; } static int locate_object(char *object_location, loose_backend *backend, const git_oid *oid) { object_file_name(object_location, GIT_PATH_MAX, backend->objects_dir, oid); return gitfo_exists(object_location); } /*********************************************************** * * LOOSE BACKEND PUBLIC API * * Implement the git_odb_backend API calls * ***********************************************************/ int loose_backend__read_header(git_rawobj *obj, git_odb_backend *backend, const git_oid *oid) { char object_path[GIT_PATH_MAX]; assert(obj && backend && oid); if (locate_object(object_path, (loose_backend *)backend, oid) < 0) return GIT_ENOTFOUND; return read_header_loose(obj, object_path); } int loose_backend__read(git_rawobj *obj, git_odb_backend *backend, const git_oid *oid) { char object_path[GIT_PATH_MAX]; assert(obj && backend && oid); if (locate_object(object_path, (loose_backend *)backend, oid) < 0) return GIT_ENOTFOUND; return read_loose(obj, object_path); } int loose_backend__exists(git_odb_backend *backend, const git_oid *oid) { char object_path[GIT_PATH_MAX]; assert(backend && oid); return locate_object(object_path, (loose_backend *)backend, oid) == GIT_SUCCESS; } int loose_backend__write(git_oid *id, git_odb_backend *_backend, git_rawobj *obj) { char hdr[64]; int hdrlen; gitfo_buf buf = GITFO_BUF_INIT; int error; loose_backend *backend; assert(id && _backend && obj); backend = (loose_backend *)_backend; if ((error = git_odb__hash_obj(id, hdr, sizeof(hdr), &hdrlen, obj)) < 0) return error; if (git_odb_exists(_backend->odb, id)) return GIT_SUCCESS; if ((error = deflate_obj(&buf, hdr, hdrlen, obj, backend->object_zlib_level)) < 0) return error; error = write_obj(&buf, id, backend); gitfo_free_buf(&buf); return error; } void loose_backend__free(git_odb_backend *_backend) { loose_backend *backend; assert(_backend); backend = (loose_backend *)_backend; free(backend->objects_dir); free(backend); } int git_odb_backend_loose(git_odb_backend **backend_out, const char *objects_dir) { loose_backend *backend; backend = git__calloc(1, sizeof(loose_backend)); if (backend == NULL) return GIT_ENOMEM; backend->objects_dir = git__strdup(objects_dir); if (backend->objects_dir == NULL) { free(backend); return GIT_ENOMEM; } backend->object_zlib_level = Z_BEST_SPEED; backend->fsync_object_files = 0; backend->parent.read = &loose_backend__read; backend->parent.read_header = &loose_backend__read_header; backend->parent.write = &loose_backend__write; backend->parent.exists = &loose_backend__exists; backend->parent.free = &loose_backend__free; *backend_out = (git_odb_backend *)backend; return GIT_SUCCESS; }