commit 81842610d679c16c5a5d624b9d815e9c64a85a95c0984a5a29110f3a36f48ce8 from: Stefan Sperling date: Tue Jul 22 08:13:47 2025 UTC fix off-by-one during deltification in maximum stretch size calculation Fix an off-by-one which causes invalid deltas to be written when common file sections exceed the maximum size which can be represented in a base-copy delta operation. This bug causes an invalid pack file to be written which neither gotadmin nor git will index successfully. Add a test which triggers the problem: got-index-pack: delta application result size mismatch: \ actual: 65536 expected: 16777216: bad delta ok op@ commit - c704b9ca5f69811fdd50e368098583094b2d45f00587ac791b0bbbaa53fc53e7 commit + 81842610d679c16c5a5d624b9d815e9c64a85a95c0984a5a29110f3a36f48ce8 blob - 4f94241158d08e7a320edc2cc4f3da7c2241d5dc0cb6ab014f77f9c99614a18c blob + 3cceecbf779c316484670d21ecd96a42eb0770d893541a0053ea0026a5a53a65 --- lib/deltify.c +++ lib/deltify.c @@ -504,7 +504,7 @@ stretchblk(FILE *basefile, off_t base_offset0, struct SEEK_SET) == -1) return got_error_from_errno("fseeko"); - while (buf_equal && *blocklen < (1 << 24) - 1) { + while (buf_equal && *blocklen < GOT_DELTIFY_STRETCHMAX - 1) { base_r = fread(basebuf, 1, sizeof(basebuf), basefile); if (base_r == 0) { if (ferror(basefile)) @@ -522,6 +522,8 @@ stretchblk(FILE *basefile, off_t base_offset0, struct buf_equal = 0; break; } + if (*blocklen >= GOT_DELTIFY_STRETCHMAX) + break; (*blocklen)++; } } @@ -558,6 +560,8 @@ stretchblk_file_mem(uint8_t *basedata, off_t base_offs buf_equal = 0; break; } + if (*blocklen >= GOT_DELTIFY_STRETCHMAX) + break; (*blocklen)++; } } @@ -598,6 +602,8 @@ stretchblk_mem_file(FILE *basefile, off_t base_offset0 buf_equal = 0; break; } + if (*blocklen >= GOT_DELTIFY_STRETCHMAX) + break; (*blocklen)++; } } @@ -629,7 +635,7 @@ stretchblk_mem_mem(uint8_t *basedata, off_t base_offse p = data + fileoffset; q = basedata + base_offset; maxlen = MIN(basefile_size - base_offset, filesize - fileoffset); - for (i = 0; i < maxlen && *blocklen < (1 << 24) - 1; i++) { + for (i = 0; i < maxlen && *blocklen < GOT_DELTIFY_STRETCHMAX - 1; i++) { if (p[i] != q[i]) break; (*blocklen)++; blob - c649778b0964dfb19b19ca19d808cda3f2c26a11810ee758331d1af183c9b6a8 blob + 0629ff6822a90c46f5c58b52c73b3d995082d2102661c01bb8b7bea51c61bd07 --- lib/got_lib_deltify.h +++ lib/got_lib_deltify.h @@ -45,6 +45,7 @@ enum { GOT_DELTIFY_MINCHUNK = 32, GOT_DELTIFY_MAXCHUNK = 8192, GOT_DELTIFY_SPLITMASK = (1 << 8) - 1, + GOT_DELTIFY_STRETCHMAX = (1 << 24) - 1, }; const struct got_error *got_deltify_init(struct got_delta_table **dt, FILE *f, blob - 736fcd6c07fba463fc7e0c8ffc32c9f72555a651170b080d08c7ea40eda80519 blob + c55e1bdf6468fd122ec25d687b9f7c027fd44cacadeb7987600f32bcc5000d83 --- regress/gotd/Makefile +++ regress/gotd/Makefile @@ -7,7 +7,8 @@ REGRESS_TARGETS=test_repo_read test_repo_read_group \ test_repo_write_protected test_repo_write_readonly \ test_email_notification test_http_notification \ test_git_interop test_email_and_http_notification \ - test_http_notification_hmac test_connection_limit + test_http_notification_hmac test_connection_limit \ + test_large_loose_objects NOOBJ=Yes CLEANFILES=gotd.conf gotd-secrets.conf @@ -252,6 +253,10 @@ prepare_test_repo_empty: ensure_root @chown ${GOTD_USER} "${GOTD_TEST_REPO}" @su -m ${GOTD_USER} -c 'env $(GOTD_TEST_ENV) sh ./prepare_test_repo.sh 1' +prepare_large_loose_objects: ensure_root prepare_test_repo + @su -m ${GOTD_USER} -c 'env $(GOTD_TEST_ENV) \ + sh ./prepare_large_loose_objects.sh' + test_repo_read: prepare_test_repo start_gotd_ro @-$(GOTD_TRAP); su ${GOTD_TEST_USER} -c \ 'env $(GOTD_TEST_ENV) sh ./repo_read.sh' @@ -350,4 +355,9 @@ test_connection_limit: prepare_test_repo start_gotd_co 'env $(GOTD_TEST_ENV) sh ./connection_limit.sh' @$(GOTD_STOP_CMD) 2>/dev/null +test_large_loose_objects: prepare_large_loose_objects start_gotd_ro + @-$(GOTD_TRAP); su ${GOTD_TEST_USER} -c \ + 'env $(GOTD_TEST_ENV) sh ./large_loose_objects.sh' + @$(GOTD_STOP_CMD) 2>/dev/null + .include blob - /dev/null blob + 3647f46de48201232b4d32302ab005f3c4d7f7dcf150a286fbefffd8a7844b97 (mode 644) --- /dev/null +++ regress/gotd/large_loose_objects.sh @@ -0,0 +1,63 @@ +#!/bin/sh +# +# Copyright (c) 2025 Stefan Sperling +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +. ../cmdline/common.sh +. ./common.sh + +test_clone_basic() { + local testroot=`test_init clone_basic 1` + + got clone -q ${GOTD_TEST_REPO_URL} $testroot/repo-clone + ret=$? + if [ $ret -ne 0 ]; then + echo "got clone failed unexpectedly" >&2 + test_done "$testroot" "1" + return 1 + fi + + # Verify that the clone operation worked fine. + git_fsck "$testroot" "$testroot/repo-clone" + ret=$? + if [ $ret -ne 0 ]; then + test_done "$testroot" "1" + return 1 + fi + + got tree -R -r "$testroot/repo-clone" > $testroot/stdout + cat > $testroot/stdout.expected < +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +test_tree=`mktemp -d "${GOTD_TEST_ROOT}/gotd-test-tree-XXXXXXXXXX"` +trap "rm -r $test_tree" HUP INT QUIT PIPE TERM + +got checkout -q "${GOTD_TEST_REPO}" "$test_tree" > /dev/null + +dd if=/dev/random of="$test_tree/large_file1" count=32768 status=none +(cd "$test_tree" && got add "$test_tree/large_file1" > /dev/null) +for i in 2 3; do + cp "$test_tree/large_file1" "$test_tree/large_file$i" + dd if=/dev/random of="$test_tree/large_file$i" seek=32768 count=64 \ + status=none + (cd "$test_tree" && got add "$test_tree/large_file$i" > /dev/null) +done + +(cd "$test_tree" && got commit -m "add large objects" "$test_tree" > /dev/null) + +rm -r "$test_tree"