Skip to content

checksum: keep the link alive while hashing a file (fixes #839) #1031

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dr-who wants to merge 1 commit into
base: master
Choose a base branch
from
Open

checksum: keep the link alive while hashing a file (fixes #839) #1031

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 42 additions & 12 deletions checksum.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@

extern int am_server;
extern int whole_file;
extern int allowed_lull;
extern int checksum_seed;
extern int protocol_version;
extern int proper_seed_order;
Expand Down Expand Up @@ -398,6 +399,41 @@ void get_checksum2(char *buf, int32 len, char *sum)
}
}

/* Hashing a large file sends nothing to the peer for as long as it takes, so a
* --checksum run over a big file looks like a dead link and the far end aborts
* once --timeout elapses. Tick the keep-alive as we go; maybe_send_keepalive()
* itself only acts once allowed_lull (half of --timeout) has passed.
*
* `n` is the size of the piece just hashed, and we only consult the clock once
* per CHUNK_SIZE of it. The MD4 loop feeds us CSUM_CHUNK (64) bytes at a time,
* so ticking per piece would call time() 512x more often for MD4 than for the
* other digests -- 16.7 million times per GiB, which is cheap on a vDSO clock
* and decidedly not on a platform where time() is a real syscall. */
static void checksum_keepalive(int32 n)
{
static int32 until_tick = CHUNK_SIZE;

if (!allowed_lull)
return;
if ((until_tick -= n) > 0)
return;
until_tick = CHUNK_SIZE;
maybe_send_keepalive(time(NULL), MSK_ALLOW_FLUSH);
}

/* Feed the mapped file to a digest in `chunk`-sized pieces. UPDATE is an
* expression using _p/_n for the piece; the digests have differing signatures,
* so an expression is what lets one loop serve all of them. Each caller keeps
* its own trailing-partial-chunk handling, which is not identical across
* digests (see the pre-27 MD4 case below). */
#define CHECKSUM_FEED(chunk, UPDATE) \
for (i = 0; i + (chunk) <= len; i += (chunk)) { \
const char *_p = map_ptr(buf, i, (chunk)); \
int32 _n = (chunk); \
UPDATE; \
checksum_keepalive(_n); \
}

void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
{
struct map_struct *buf;
Expand All @@ -421,8 +457,7 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)

EVP_DigestInit_ex(evp, file_sum_evp_md, NULL);

for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE)
EVP_DigestUpdate(evp, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE);
CHECKSUM_FEED(CHUNK_SIZE, EVP_DigestUpdate(evp, (uchar *)_p, _n));

remainder = (int32)(len - i);
if (remainder > 0)
Expand All @@ -440,8 +475,7 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)

XXH64_reset(state, 0);

for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE)
XXH64_update(state, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE);
CHECKSUM_FEED(CHUNK_SIZE, XXH64_update(state, (uchar *)_p, _n));

remainder = (int32)(len - i);
if (remainder > 0)
Expand All @@ -459,8 +493,7 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)

XXH3_64bits_reset(state);

for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE)
XXH3_64bits_update(state, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE);
CHECKSUM_FEED(CHUNK_SIZE, XXH3_64bits_update(state, (uchar *)_p, _n));

remainder = (int32)(len - i);
if (remainder > 0)
Expand All @@ -477,8 +510,7 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)

XXH3_128bits_reset(state);

for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE)
XXH3_128bits_update(state, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE);
CHECKSUM_FEED(CHUNK_SIZE, XXH3_128bits_update(state, (uchar *)_p, _n));

remainder = (int32)(len - i);
if (remainder > 0)
Expand All @@ -495,8 +527,7 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)

md5_begin(&m5);

for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE)
md5_update(&m5, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE);
CHECKSUM_FEED(CHUNK_SIZE, md5_update(&m5, (uchar *)_p, _n));

remainder = (int32)(len - i);
if (remainder > 0)
Expand All @@ -513,8 +544,7 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)

mdfour_begin(&m);

for (i = 0; i + CSUM_CHUNK <= len; i += CSUM_CHUNK)
mdfour_update(&m, (uchar *)map_ptr(buf, i, CSUM_CHUNK), CSUM_CHUNK);
CHECKSUM_FEED(CSUM_CHUNK, mdfour_update(&m, (uchar *)_p, _n));

/* Prior to version 27 an incorrect MD4 checksum was computed
* by failing to call mdfour_tail() for block sizes that
Expand Down
Loading