diff --git a/core/src/main/java/tech/ydb/core/keygen/BaseKeyGen.java b/core/src/main/java/tech/ydb/core/keygen/BaseKeyGen.java new file mode 100644 index 000000000..89032168f --- /dev/null +++ b/core/src/main/java/tech/ydb/core/keygen/BaseKeyGen.java @@ -0,0 +1,184 @@ +package tech.ydb.core.keygen; + +import java.nio.ByteBuffer; +import java.security.SecureRandom; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Base64; +import java.util.UUID; + +/** + * Implementation helpers for key generators. + * + * @author zinal + */ +public class BaseKeyGen { + + /** + * Bit width of the embedded second-precision timestamp field. + */ + static final int TIMESTAMP_BITS = 30; + + /** + * Number of seconds which fit within the bit width specified. + */ + static final long TIMESTAMP_SECONDS = (1L << TIMESTAMP_BITS); + + /** + * Low bit index (inclusive) of the 30-bit timestamp field when + * {@code maskPos == 0} (1-bit prefix). Shifts down with larger prefixes. + */ + static final int TIMESTAMP_FIELD_LOW_BIT = 33; + + /** + * A position within an array of pre-computed bitmasks to be used. + */ + protected final int maskPos; + + protected BaseKeyGen(int prefixBits) { + if (prefixBits < 1 || prefixBits > 18) { + throw new IllegalArgumentException("Unsupported prefix length: " + prefixBits); + } + this.maskPos = prefixBits - 1; + } + + /** + * @return Prefix size used for construction, in bits. + */ + public int getPrefixBits() { + return maskPos + 1; + } + + /** + * @return Prefix mask to be applied + */ + public long getPrefixMask() { + return Holder.PREFIX_MASKS[maskPos]; + } + + /** + * Generates the new shared prefix to generate a series of related IDs. + * + * @return Random value to be used as a prefix. + */ + public long nextPrefix() { + final SecureRandom sr = Holder.SR; + byte[] data = new byte[8]; + sr.nextBytes(data); + long lsb = 0; + for (int i = 0; i < 8; i++) { + lsb = (lsb << 8) | (data[i] & 0xff); + } + return lsb; + } + + protected final long update(long msb, long prefix, Instant instant) { + long tsMask = Holder.TS_MASKS[maskPos]; + long tsCode = getTimestampCode(instant); + tsCode = tsCode << (TIMESTAMP_FIELD_LOW_BIT - maskPos); + long bits; + if (prefix == -1L) { + bits = msb & ~tsMask; + bits |= tsCode & tsMask; + } else { + long prefixMask = Holder.PREFIX_MASKS[maskPos]; + bits = msb & ~(prefixMask | tsMask); + bits |= (prefix & prefixMask) | (tsCode & tsMask); + } + return bits; + } + + /** + * Computes the second-precision timestamp code: seconds since + * 2020-01-01T00:00:00Z, in the range {@code [0, 2^30)}. + * + * @param instant the instant to be used + * @return timestamp code between 0 and 2^30 - 1, inclusive + */ + public static int getTimestampCode(Instant instant) { + Instant sec = instant.truncatedTo(ChronoUnit.SECONDS); + long diff = sec.getEpochSecond() % TIMESTAMP_SECONDS; + if (diff < 0) { + throw new IllegalArgumentException( + "Instant out of 30-bit timestamp range: " + instant); + } + return (int) diff; + } + + /** + * YDB uses GUID (Microsoft-style) mixed-endian format. + * + *
+ * xxxxxxxx 0 1 2 3x 4 5 6 7 + * INPUT: 01020304 05060708 090a0b0c 0d0e0f10 + * OUTPUT: 04030201 06050807 090a0b0c 0d0e0f10 + *+ * + * This function puts the bytes of MSB in the proper order. + * + * @param v MSB value in standard big-endian long representation + * @return MSB value with byte order adjusted for YDB GUID storage + */ + public static long reorder(long v) { + long b0 = (v >>> 56) & 0xffL; + long b1 = (v >>> 48) & 0xffL; + long b2 = (v >>> 40) & 0xffL; + long b3 = (v >>> 32) & 0xffL; + long b4 = (v >>> 24) & 0xffL; + long b5 = (v >>> 16) & 0xffL; + long b6 = (v >>> 8) & 0xffL; + long b7 = v & 0xffL; + return (b3 << 56) | (b2 << 48) | (b1 << 40) | (b0 << 32) + | (b5 << 24) | (b4 << 16) | (b7 << 8) | b6; + } + + public static long updateVersion(long msb) { + return (msb & ~0xF000L) | 0x8000L; + } + + public static long updateVariant(long lsb) { + return (lsb & 0x3FFFFFFFFFFFFFFFL) + | 0x8000000000000000L; + } + + /** + * Convert a UUID value to a base64 text representation. + * + * @param uuid Value to be converted + * @return Text representation of a UUID value of a fixed length 22 symbols + */ + public static String toString(UUID uuid) { + // apply byte swaps to restore the "regular" ordering + long msb = reorder(uuid.getMostSignificantBits()); + ByteBuffer byteArray = ByteBuffer.allocate(16); + byteArray.putLong(msb); + byteArray.putLong(uuid.getLeastSignificantBits()); + return Base64.getUrlEncoder() + .encodeToString(byteArray.array()) + .substring(0, 22); + } + + /** + * A holder class to defer initialization until needed. + */ + static class Holder { + + static final SecureRandom SR = new SecureRandom(); + + static final long[] PREFIX_MASKS; + static final long[] TS_MASKS; + + static { + long[] pf = new long[32]; + long[] ts = new long[32]; + pf[0] = 0x8000000000000000L; + ts[0] = (((1L << TIMESTAMP_BITS) - 1) << TIMESTAMP_FIELD_LOW_BIT); + for (int i = 1; i < 32; ++i) { + pf[i] = pf[i - 1] | (1L << (63 - i)); + ts[i] = (ts[i - 1] >>> 1); + } + PREFIX_MASKS = pf; + TS_MASKS = ts; + } + } +} diff --git a/core/src/main/java/tech/ydb/core/keygen/TextKeyGen.java b/core/src/main/java/tech/ydb/core/keygen/TextKeyGen.java new file mode 100644 index 000000000..7cdb16a86 --- /dev/null +++ b/core/src/main/java/tech/ydb/core/keygen/TextKeyGen.java @@ -0,0 +1,118 @@ +package tech.ydb.core.keygen; + +import java.nio.ByteBuffer; +import java.security.SecureRandom; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.util.Base64; + +/** + * Random text-format ID generator creates cache friendly identifiers to be used + * as primary keys for YDB row-organized tables. + * + * Similar to UuidKeyGen, but for keys in textual (base64-encoded) format. + * + * An extra difference is that, unlike UuidKeyGen, TextKeyGen does not set the + * UUID version and variant bits, so the generated value is formally not a UUID. + * + * @author zinal + */ +public class TextKeyGen extends BaseKeyGen { + + /** + * Constructs the generator instance with the default prefix size of 10 + * bits. + * + * Works best for up to 1k table partitions. + */ + public TextKeyGen() { + super(10); + } + + /** + * Constructs the generator instance with the custom prefix size. + * + * @param prefixBits Number of bits for the prefix, 1 to 18 bits. + */ + public TextKeyGen(int prefixBits) { + super(prefixBits); + } + + /** + * Generates the new ID with the specified prefix and instant (second + * precision for the embedded timestamp field). + * + * @param prefix Prefix value + * @param instant The instant whose second is embedded in the ID + * @return Base64 encoded ID with the embedded prefix and timestamp. + */ + public String nextValue(long prefix, Instant instant) { + SecureRandom sr = Holder.SR; + byte[] data = new byte[16]; + sr.nextBytes(data); + + long msb = 0; + for (int i = 0; i < 8; i++) { + msb = (msb << 8) | (data[i] & 0xff); + } + msb = update(msb, prefix, instant); + ByteBuffer.wrap(data).putLong(msb); + return Base64.getUrlEncoder() + .encodeToString(data) + .substring(0, 22); + } + + /** + * Generates the new ID with the specified prefix value. + * + * @param prefix Prefix value + * @param date The date (UTC midnight) used for the embedded timestamp + * @return Base64 encoded ID with the embedded prefix and timestamp for the + * start of date specified. + */ + public String nextValue(long prefix, LocalDate date) { + return nextValue(prefix, date.atStartOfDay(ZoneOffset.UTC).toInstant()); + } + + /** + * Generates the new ID with the specified prefix value. + * + * @param prefix Prefix value + * @return Base64 encoded ID with the embedded prefix and the current + * timestamp. + */ + public String nextValue(long prefix) { + return nextValue(prefix, Instant.now()); + } + + /** + * Generates the new ID with the specified instant. + * + * @param instant Timestamp for the value being generated. + * @return Base64 encoded ID with the embedded timestamp code. + */ + public String nextValue(Instant instant) { + return nextValue(-1L, instant); + } + + /** + * Generates the new ID with the random prefix value and a specified date. + * + * @param date The date (UTC midnight) used for the embedded timestamp + * @return Base64 encoded ID with the embedded timestamp for the start of + * date specified. + */ + public String nextValue(LocalDate date) { + return nextValue(-1L, date); + } + + /** + * Generates the new ID with the random prefix value. + * + * @return Base64 encoded ID with the embedded current timestamp. + */ + public String nextValue() { + return nextValue(-1L, Instant.now()); + } +} diff --git a/core/src/main/java/tech/ydb/core/keygen/UuidKeyGen.java b/core/src/main/java/tech/ydb/core/keygen/UuidKeyGen.java new file mode 100644 index 000000000..befbcaec0 --- /dev/null +++ b/core/src/main/java/tech/ydb/core/keygen/UuidKeyGen.java @@ -0,0 +1,132 @@ +package tech.ydb.core.keygen; + +import java.security.SecureRandom; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.util.UUID; + +/** + * Random UUID generator creates cache friendly identifiers to be used as + * primary keys for YDB row-organized tables. + * + * Each generated value consists of a random prefix of the specified length, a + * second-precision timestamp code of 30 bits, followed by the random suffix. + * + * The timestamp is UNIX epoch seconds, modulo 2^30 (about 34 years), which + * avoids code reuse for well over 30 years from the overlap. + * + * In addition, the generator supports the "fixed prefix" schema, in which a + * common prefix value is used for a series of related ids generated (typically + * to be written in a single transaction). + * + * The generated value is optimized for the actual internal format of UUID + * storage in YDB, which uses GUID (Microsoft-style) mixed-endian byte ordering. + * This affects the actual ordering, so it is important to properly put the + * bytes in the correct order. + * + * @author zinal + */ +public class UuidKeyGen extends BaseKeyGen { + + /** + * Constructs the generator instance with the default prefix size of 10 + * bits. + * + * Works best for up to 1k table partitions. + */ + public UuidKeyGen() { + super(10); + } + + /** + * Constructs the generator instance with the custom prefix size. + * + * @param prefixBits Number of bits for the prefix, 1 to 18 bits. + */ + public UuidKeyGen(int prefixBits) { + super(prefixBits); + } + + /** + * Generates the new ID with the specified prefix value and calendar date + * (UTC midnight). + * + * @param prefix Prefix value + * @param date The date whose start-of-day UTC instant is embedded + * @return Random UUID with the embedded prefix, timestamp code and suffix. + */ + public UUID nextValue(long prefix, LocalDate date) { + return nextValue(prefix, date.atStartOfDay(ZoneOffset.UTC).toInstant()); + } + + /** + * Generates the new ID with the specified prefix value and instant + * (truncated to whole seconds for the embedded timestamp field). + * + * @param prefix Prefix value + * @param instant The instant whose second is embedded in the UUID + * @return Random UUID with the embedded prefix, timestamp code and suffix. + */ + public UUID nextValue(long prefix, Instant instant) { + SecureRandom sr = Holder.SR; + byte[] data = new byte[16]; + sr.nextBytes(data); + + long msb = 0; + long lsb = 0; + for (int i = 0; i < 8; i++) { + msb = (msb << 8) | (data[i] & 0xff); + } + for (int i = 8; i < 16; i++) { + lsb = (lsb << 8) | (data[i] & 0xff); + } + + msb = update(msb, prefix, instant); + msb = reorder(msb); + msb = updateVersion(msb); + lsb = updateVariant(lsb); + return new UUID(msb, lsb); + } + + /** + * Generates the new ID with the specified prefix value. + * + * @param prefix Prefix value + * @return Random UUID with the embedded prefix, timestamp code and suffix. + */ + public UUID nextValue(long prefix) { + return nextValue(prefix, Instant.now()); + } + + /** + * Generates the new ID with the random prefix value and a specified + * instant. + * + * @param instant Timestamp for the value being generated. + * @return Random UUID with the embedded prefix, timestamp code and suffix. + */ + public UUID nextValue(Instant instant) { + return nextValue(-1L, instant); + } + + /** + * Generates the new ID with the random prefix value and a specified date. + * + * @param date The date (UTC midnight) used for the embedded timestamp + * @return Random UUID with the embedded prefix, timestamp code and suffix. + */ + public UUID nextValue(LocalDate date) { + return nextValue(-1L, date); + } + + /** + * Generates the new ID with the random prefix value. + * + * @return Random UUID with the embedded prefix, timestamp code and suffix. + */ + public UUID nextValue() { + return nextValue(-1L, Instant.now()); + } + +} diff --git a/core/src/test/java/tech/ydb/core/keygen/ConvertToYdbTest.java b/core/src/test/java/tech/ydb/core/keygen/ConvertToYdbTest.java new file mode 100644 index 000000000..30ebaeb4f --- /dev/null +++ b/core/src/test/java/tech/ydb/core/keygen/ConvertToYdbTest.java @@ -0,0 +1,23 @@ +package tech.ydb.core.keygen; + +import java.nio.ByteBuffer; +import org.junit.Assert; +import org.junit.Test; + +/** + * + * @author zinal + */ +public class ConvertToYdbTest { + + @Test + public void convertToYdbTest() { + byte[] input = new byte[]{1, 2, 3, 4, 5, 6, 7, 8}; + long v0 = ByteBuffer.wrap(input).getLong(); + long v1 = UuidKeyGen.reorder(v0); + long v2 = UuidKeyGen.reorder(v1); + Assert.assertEquals(v0, v2); + Assert.assertNotEquals(v0, v1); + } + +} diff --git a/core/src/test/java/tech/ydb/core/keygen/KeyGenTestSupport.java b/core/src/test/java/tech/ydb/core/keygen/KeyGenTestSupport.java new file mode 100644 index 000000000..20d97f385 --- /dev/null +++ b/core/src/test/java/tech/ydb/core/keygen/KeyGenTestSupport.java @@ -0,0 +1,66 @@ +package tech.ydb.core.keygen; + +import java.util.Base64; +import java.util.UUID; + +/** + * Shared helpers for {@link UuidKeyGen} and {@link TextKeyGen} tests. + */ +final class KeyGenTestSupport { + + private KeyGenTestSupport() { + } + + static long timestampMask(int prefixBits) { + int maskPos = prefixBits - 1; + return (((1L << BaseKeyGen.TIMESTAMP_BITS) - 1L) << BaseKeyGen.TIMESTAMP_FIELD_LOW_BIT) >>> maskPos; + } + + static long logicalMsbFromUuid(UUID uuid) { + return BaseKeyGen.reorder(uuid.getMostSignificantBits()); + } + + static long logicalMsbFromTextKey(String key) { + return bytesFromTextKey(key).getLong(0); + } + + static long logicalLsbFromTextKey(String key) { + return bytesFromTextKey(key).getLong(8); + } + + private static java.nio.ByteBuffer bytesFromTextKey(String key) { + String padded = key; + while (padded.length() % 4 != 0) { + padded += "="; + } + byte[] data = Base64.getUrlDecoder().decode(padded); + if (data.length != 16) { + throw new IllegalArgumentException("Expected 16 bytes, got " + data.length); + } + return java.nio.ByteBuffer.wrap(data); + } + + static long extractEmbeddedPrefix(long logicalMsb, BaseKeyGen generator) { + return logicalMsb & generator.getPrefixMask(); + } + + static int extractEmbeddedTimestampCode(long logicalMsb, int prefixBits) { + long tsMask = timestampMask(prefixBits); + int shift = BaseKeyGen.TIMESTAMP_FIELD_LOW_BIT - (prefixBits - 1); + return (int) ((logicalMsb & tsMask) >>> shift); + } + + static int logicalVersionNibble(long logicalMsb) { + return (int) ((logicalMsb >>> 4) & 0x0f); + } + + static int logicalVariantBits(long logicalLsb) { + return (int) ((logicalLsb >>> 56) & 0xc0); + } + + static UUID uuidFromTextKey(String key) { + return new UUID( + BaseKeyGen.reorder(KeyGenTestSupport.logicalMsbFromTextKey(key)), + KeyGenTestSupport.logicalLsbFromTextKey(key)); + } +} diff --git a/core/src/test/java/tech/ydb/core/keygen/TextKeyGenTest.java b/core/src/test/java/tech/ydb/core/keygen/TextKeyGenTest.java new file mode 100644 index 000000000..770a42db0 --- /dev/null +++ b/core/src/test/java/tech/ydb/core/keygen/TextKeyGenTest.java @@ -0,0 +1,238 @@ +package tech.ydb.core.keygen; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.util.Base64; +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; +import org.junit.Assert; +import org.junit.Test; + +/** + * Correctness tests for {@link TextKeyGen}. + */ +public class TextKeyGenTest { + + private static final Instant FIXED_INSTANT = Instant.parse("2024-06-15T12:34:56Z"); + private static final LocalDate FIXED_DATE = LocalDate.of(2024, 6, 15); + private static final long FIXED_PREFIX = 0x2A5L; + + @Test + public void defaultConstructorUsesTenPrefixBits() { + TextKeyGen generator = new TextKeyGen(); + Assert.assertEquals(10, generator.getPrefixBits()); + Assert.assertEquals(0xFFC0000000000000L, generator.getPrefixMask()); + } + + @Test + public void customPrefixBitsExposeMask() { + TextKeyGen oneBit = new TextKeyGen(1); + TextKeyGen eighteenBit = new TextKeyGen(18); + + Assert.assertEquals(1, oneBit.getPrefixBits()); + Assert.assertEquals(0x8000000000000000L, oneBit.getPrefixMask()); + Assert.assertEquals(18, eighteenBit.getPrefixBits()); + Assert.assertEquals(0xFFFFC00000000000L, eighteenBit.getPrefixMask()); + } + + @Test + public void unsupportedPrefixLengthIsRejected() { + try { + new TextKeyGen(0); + Assert.fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException ex) { + Assert.assertEquals("Unsupported prefix length: 0", ex.getMessage()); + } + + try { + new TextKeyGen(19); + Assert.fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException ex) { + Assert.assertEquals("Unsupported prefix length: 19", ex.getMessage()); + } + } + + @Test + public void nextPrefixProducesDistinctValues() { + TextKeyGen generator = new TextKeyGen(12); + Set