From 5eab8bc0df8a688656a83182d9a1f375c5dadff8 Mon Sep 17 00:00:00 2001 From: jchmura Date: Sat, 15 Aug 2026 19:22:28 +0000 Subject: [PATCH] Add edge quantization metadata proto --- .../research/gbml/preprocessed_metadata.proto | 2 + .../PreprocessedMetadata.scala | 45 ++++++++++++++++--- .../PreprocessedMetadataProto.scala | 34 +++++++------- .../PreprocessedMetadata.scala | 45 ++++++++++++++++--- .../PreprocessedMetadataProto.scala | 34 +++++++------- .../gbml/preprocessed_metadata_pb2.py | 18 ++++---- .../gbml/preprocessed_metadata_pb2.pyi | 9 +++- 7 files changed, 134 insertions(+), 53 deletions(-) diff --git a/proto/snapchat/research/gbml/preprocessed_metadata.proto b/proto/snapchat/research/gbml/preprocessed_metadata.proto index d7dfe3469..661c91d85 100644 --- a/proto/snapchat/research/gbml/preprocessed_metadata.proto +++ b/proto/snapchat/research/gbml/preprocessed_metadata.proto @@ -71,6 +71,8 @@ message PreprocessedMetadata{ optional uint32 feature_dim = 6; // Contains categorical feature vocabularies string transform_fn_assets_uri = 7; + // Optional quantized main-edge feature metadata. + FeatureQuantizationMetadata quantized_feature_metadata = 8; } // Houses metadata about edge TFTransform output from DataPreprocessor. diff --git a/scala/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadata.scala b/scala/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadata.scala index 7a9012ffa..9ddc933a7 100644 --- a/scala/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadata.scala +++ b/scala/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadata.scala @@ -1125,6 +1125,8 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r * Feature dimension after preprocessing * @param transformFnAssetsUri * Contains categorical feature vocabularies + * @param quantizedFeatureMetadata + * Optional quantized main-edge feature metadata. */ @SerialVersionUID(0L) final case class EdgeMetadataInfo( @@ -1135,6 +1137,7 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r enumeratedEdgeDataBqTable: _root_.scala.Predef.String = "", featureDim: _root_.scala.Option[_root_.scala.Int] = _root_.scala.None, transformFnAssetsUri: _root_.scala.Predef.String = "", + quantizedFeatureMetadata: _root_.scala.Option[snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata] = _root_.scala.None, unknownFields: _root_.scalapb.UnknownFieldSet = _root_.scalapb.UnknownFieldSet.empty ) extends scalapb.GeneratedMessage with scalapb.lenses.Updatable[EdgeMetadataInfo] { @transient @@ -1181,6 +1184,10 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(7, __value) } }; + if (quantizedFeatureMetadata.isDefined) { + val __value = quantizedFeatureMetadata.get + __size += 1 + _root_.com.google.protobuf.CodedOutputStream.computeUInt32SizeNoTag(__value.serializedSize) + __value.serializedSize + }; __size += unknownFields.serializedSize __size } @@ -1230,6 +1237,12 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r _output__.writeString(7, __v) } }; + quantizedFeatureMetadata.foreach { __v => + val __m = __v + _output__.writeTag(8, 2) + _output__.writeUInt32NoTag(__m.serializedSize) + __m.writeTo(_output__) + }; unknownFields.writeTo(_output__) } def clearFeatureKeys = copy(featureKeys = _root_.scala.Seq.empty) @@ -1247,6 +1260,9 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r def clearFeatureDim: EdgeMetadataInfo = copy(featureDim = _root_.scala.None) def withFeatureDim(__v: _root_.scala.Int): EdgeMetadataInfo = copy(featureDim = Option(__v)) def withTransformFnAssetsUri(__v: _root_.scala.Predef.String): EdgeMetadataInfo = copy(transformFnAssetsUri = __v) + def getQuantizedFeatureMetadata: snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata = quantizedFeatureMetadata.getOrElse(snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata.defaultInstance) + def clearQuantizedFeatureMetadata: EdgeMetadataInfo = copy(quantizedFeatureMetadata = _root_.scala.None) + def withQuantizedFeatureMetadata(__v: snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata): EdgeMetadataInfo = copy(quantizedFeatureMetadata = Option(__v)) def withUnknownFields(__v: _root_.scalapb.UnknownFieldSet) = copy(unknownFields = __v) def discardUnknownFields = copy(unknownFields = _root_.scalapb.UnknownFieldSet.empty) def getFieldByNumber(__fieldNumber: _root_.scala.Int): _root_.scala.Any = { @@ -1270,6 +1286,7 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r val __t = transformFnAssetsUri if (__t != "") __t else null } + case 8 => quantizedFeatureMetadata.orNull } } def getField(__field: _root_.scalapb.descriptors.FieldDescriptor): _root_.scalapb.descriptors.PValue = { @@ -1282,6 +1299,7 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r case 5 => _root_.scalapb.descriptors.PString(enumeratedEdgeDataBqTable) case 6 => featureDim.map(_root_.scalapb.descriptors.PInt(_)).getOrElse(_root_.scalapb.descriptors.PEmpty) case 7 => _root_.scalapb.descriptors.PString(transformFnAssetsUri) + case 8 => quantizedFeatureMetadata.map(_.toPMessage).getOrElse(_root_.scalapb.descriptors.PEmpty) } } def toProtoString: _root_.scala.Predef.String = _root_.scalapb.TextFormat.printToUnicodeString(this) @@ -1299,6 +1317,7 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r var __enumeratedEdgeDataBqTable: _root_.scala.Predef.String = "" var __featureDim: _root_.scala.Option[_root_.scala.Int] = _root_.scala.None var __transformFnAssetsUri: _root_.scala.Predef.String = "" + var __quantizedFeatureMetadata: _root_.scala.Option[snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata] = _root_.scala.None var `_unknownFields__`: _root_.scalapb.UnknownFieldSet.Builder = null var _done__ = false while (!_done__) { @@ -1319,6 +1338,8 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r __featureDim = Option(_input__.readUInt32()) case 58 => __transformFnAssetsUri = _input__.readStringRequireUtf8() + case 66 => + __quantizedFeatureMetadata = Option(__quantizedFeatureMetadata.fold(_root_.scalapb.LiteParser.readMessage[snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata](_input__))(_root_.scalapb.LiteParser.readMessage(_input__, _))) case tag => if (_unknownFields__ == null) { _unknownFields__ = new _root_.scalapb.UnknownFieldSet.Builder() @@ -1334,6 +1355,7 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r enumeratedEdgeDataBqTable = __enumeratedEdgeDataBqTable, featureDim = __featureDim, transformFnAssetsUri = __transformFnAssetsUri, + quantizedFeatureMetadata = __quantizedFeatureMetadata, unknownFields = if (_unknownFields__ == null) _root_.scalapb.UnknownFieldSet.empty else _unknownFields__.result() ) } @@ -1347,13 +1369,20 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r schemaUri = __fieldsMap.get(scalaDescriptor.findFieldByNumber(4).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), enumeratedEdgeDataBqTable = __fieldsMap.get(scalaDescriptor.findFieldByNumber(5).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), featureDim = __fieldsMap.get(scalaDescriptor.findFieldByNumber(6).get).flatMap(_.as[_root_.scala.Option[_root_.scala.Int]]), - transformFnAssetsUri = __fieldsMap.get(scalaDescriptor.findFieldByNumber(7).get).map(_.as[_root_.scala.Predef.String]).getOrElse("") + transformFnAssetsUri = __fieldsMap.get(scalaDescriptor.findFieldByNumber(7).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), + quantizedFeatureMetadata = __fieldsMap.get(scalaDescriptor.findFieldByNumber(8).get).flatMap(_.as[_root_.scala.Option[snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata]]) ) case _ => throw new RuntimeException("Expected PMessage") } def javaDescriptor: _root_.com.google.protobuf.Descriptors.Descriptor = snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.javaDescriptor.getNestedTypes().get(4) def scalaDescriptor: _root_.scalapb.descriptors.Descriptor = snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.scalaDescriptor.nestedMessages(4) - def messageCompanionForFieldNumber(__number: _root_.scala.Int): _root_.scalapb.GeneratedMessageCompanion[_] = throw new MatchError(__number) + def messageCompanionForFieldNumber(__number: _root_.scala.Int): _root_.scalapb.GeneratedMessageCompanion[_] = { + var __out: _root_.scalapb.GeneratedMessageCompanion[_] = null + (__number: @_root_.scala.unchecked) match { + case 8 => __out = snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata + } + __out + } lazy val nestedMessagesCompanions: Seq[_root_.scalapb.GeneratedMessageCompanion[_ <: _root_.scalapb.GeneratedMessage]] = Seq.empty def enumCompanionForFieldNumber(__fieldNumber: _root_.scala.Int): _root_.scalapb.GeneratedEnumCompanion[_] = throw new MatchError(__fieldNumber) lazy val defaultInstance = snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.EdgeMetadataInfo( @@ -1363,7 +1392,8 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r schemaUri = "", enumeratedEdgeDataBqTable = "", featureDim = _root_.scala.None, - transformFnAssetsUri = "" + transformFnAssetsUri = "", + quantizedFeatureMetadata = _root_.scala.None ) implicit class EdgeMetadataInfoLens[UpperPB](_l: _root_.scalapb.lenses.Lens[UpperPB, snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.EdgeMetadataInfo]) extends _root_.scalapb.lenses.ObjectLens[UpperPB, snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.EdgeMetadataInfo](_l) { def featureKeys: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Seq[_root_.scala.Predef.String]] = field(_.featureKeys)((c_, f_) => c_.copy(featureKeys = f_)) @@ -1374,6 +1404,8 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r def featureDim: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Int] = field(_.getFeatureDim)((c_, f_) => c_.copy(featureDim = Option(f_))) def optionalFeatureDim: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Option[_root_.scala.Int]] = field(_.featureDim)((c_, f_) => c_.copy(featureDim = f_)) def transformFnAssetsUri: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.transformFnAssetsUri)((c_, f_) => c_.copy(transformFnAssetsUri = f_)) + def quantizedFeatureMetadata: _root_.scalapb.lenses.Lens[UpperPB, snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata] = field(_.getQuantizedFeatureMetadata)((c_, f_) => c_.copy(quantizedFeatureMetadata = Option(f_))) + def optionalQuantizedFeatureMetadata: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Option[snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata]] = field(_.quantizedFeatureMetadata)((c_, f_) => c_.copy(quantizedFeatureMetadata = f_)) } final val FEATURE_KEYS_FIELD_NUMBER = 1 final val LABEL_KEYS_FIELD_NUMBER = 2 @@ -1382,6 +1414,7 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r final val ENUMERATED_EDGE_DATA_BQ_TABLE_FIELD_NUMBER = 5 final val FEATURE_DIM_FIELD_NUMBER = 6 final val TRANSFORM_FN_ASSETS_URI_FIELD_NUMBER = 7 + final val QUANTIZED_FEATURE_METADATA_FIELD_NUMBER = 8 def of( featureKeys: _root_.scala.Seq[_root_.scala.Predef.String], labelKeys: _root_.scala.Seq[_root_.scala.Predef.String], @@ -1389,7 +1422,8 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r schemaUri: _root_.scala.Predef.String, enumeratedEdgeDataBqTable: _root_.scala.Predef.String, featureDim: _root_.scala.Option[_root_.scala.Int], - transformFnAssetsUri: _root_.scala.Predef.String + transformFnAssetsUri: _root_.scala.Predef.String, + quantizedFeatureMetadata: _root_.scala.Option[snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata] ): _root_.snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.EdgeMetadataInfo = _root_.snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.EdgeMetadataInfo( featureKeys, labelKeys, @@ -1397,7 +1431,8 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r schemaUri, enumeratedEdgeDataBqTable, featureDim, - transformFnAssetsUri + transformFnAssetsUri, + quantizedFeatureMetadata ) // @@protoc_insertion_point(GeneratedMessageCompanion[snapchat.research.gbml.PreprocessedMetadata.EdgeMetadataInfo]) } diff --git a/scala/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadataProto.scala b/scala/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadataProto.scala index ad80de0ad..998cadd75 100644 --- a/scala/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadataProto.scala +++ b/scala/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadataProto.scala @@ -14,7 +14,7 @@ object PreprocessedMetadataProto extends _root_.scalapb.GeneratedFileObject { private lazy val ProtoBytes: _root_.scala.Array[Byte] = scalapb.Encoding.fromBase64(scala.collection.immutable.Seq( """CjJzbmFwY2hhdC9yZXNlYXJjaC9nYm1sL3ByZXByb2Nlc3NlZF9tZXRhZGF0YS5wcm90bxIWc25hcGNoYXQucmVzZWFyY2guZ - 2JtbCL9GgoUUHJlcHJvY2Vzc2VkTWV0YWRhdGES5gEKLGNvbmRlbnNlZF9ub2RlX3R5cGVfdG9fcHJlcHJvY2Vzc2VkX21ldGFkY + 2JtbCKlHAoUUHJlcHJvY2Vzc2VkTWV0YWRhdGES5gEKLGNvbmRlbnNlZF9ub2RlX3R5cGVfdG9fcHJlcHJvY2Vzc2VkX21ldGFkY XRhGAEgAygLMlkuc25hcGNoYXQucmVzZWFyY2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5Db25kZW5zZWROb2RlVHlwZVRvU HJlcHJvY2Vzc2VkTWV0YWRhdGFFbnRyeUIs4j8pEidjb25kZW5zZWROb2RlVHlwZVRvUHJlcHJvY2Vzc2VkTWV0YWRhdGFSJ2Nvb mRlbnNlZE5vZGVUeXBlVG9QcmVwcm9jZXNzZWRNZXRhZGF0YRLmAQosY29uZGVuc2VkX2VkZ2VfdHlwZV90b19wcmVwcm9jZXNzZ @@ -41,26 +41,28 @@ object PreprocessedMetadataProto extends _root_.scalapb.GeneratedFileObject { hR0cmFuc2Zvcm1GbkFzc2V0c1VyaVIUdHJhbnNmb3JtRm5Bc3NldHNVcmkSpQEKGnF1YW50aXplZF9mZWF0dXJlX21ldGFkYXRhG AogASgLMkguc25hcGNoYXQucmVzZWFyY2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5GZWF0dXJlUXVhbnRpemF0aW9uTWV0Y WRhdGFCHeI/GhIYcXVhbnRpemVkRmVhdHVyZU1ldGFkYXRhUhhxdWFudGl6ZWRGZWF0dXJlTWV0YWRhdGFCDgoMX2ZlYXR1cmVfZ - GltGugDChBFZGdlTWV0YWRhdGFJbmZvEjMKDGZlYXR1cmVfa2V5cxgBIAMoCUIQ4j8NEgtmZWF0dXJlS2V5c1ILZmVhdHVyZUtle + GltGpAFChBFZGdlTWV0YWRhdGFJbmZvEjMKDGZlYXR1cmVfa2V5cxgBIAMoCUIQ4j8NEgtmZWF0dXJlS2V5c1ILZmVhdHVyZUtle XMSLQoKbGFiZWxfa2V5cxgCIAMoCUIO4j8LEglsYWJlbEtleXNSCWxhYmVsS2V5cxJGChN0ZnJlY29yZF91cmlfcHJlZml4GAMgA SgJQhbiPxMSEXRmcmVjb3JkVXJpUHJlZml4UhF0ZnJlY29yZFVyaVByZWZpeBItCgpzY2hlbWFfdXJpGAQgASgJQg7iPwsSCXNja GVtYVVyaVIJc2NoZW1hVXJpEmAKHWVudW1lcmF0ZWRfZWRnZV9kYXRhX2JxX3RhYmxlGAUgASgJQh7iPxsSGWVudW1lcmF0ZWRFZ GdlRGF0YUJxVGFibGVSGWVudW1lcmF0ZWRFZGdlRGF0YUJxVGFibGUSNQoLZmVhdHVyZV9kaW0YBiABKA1CD+I/DBIKZmVhdHVyZ URpbUgAUgpmZWF0dXJlRGltiAEBElAKF3RyYW5zZm9ybV9mbl9hc3NldHNfdXJpGAcgASgJQhniPxYSFHRyYW5zZm9ybUZuQXNzZ - XRzVXJpUhR0cmFuc2Zvcm1GbkFzc2V0c1VyaUIOCgxfZmVhdHVyZV9kaW0awgQKEkVkZ2VNZXRhZGF0YU91dHB1dBI4Cg9zcmNfb - m9kZV9pZF9rZXkYASABKAlCEeI/DhIMc3JjTm9kZUlkS2V5UgxzcmNOb2RlSWRLZXkSOAoPZHN0X25vZGVfaWRfa2V5GAIgASgJQ - hHiPw4SDGRzdE5vZGVJZEtleVIMZHN0Tm9kZUlkS2V5EnYKDm1haW5fZWRnZV9pbmZvGAMgASgLMj0uc25hcGNoYXQucmVzZWFyY - 2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5FZGdlTWV0YWRhdGFJbmZvQhHiPw4SDG1haW5FZGdlSW5mb1IMbWFpbkVkZ2VJb - mZvEocBChJwb3NpdGl2ZV9lZGdlX2luZm8YBCABKAsyPS5zbmFwY2hhdC5yZXNlYXJjaC5nYm1sLlByZXByb2Nlc3NlZE1ldGFkY - XRhLkVkZ2VNZXRhZGF0YUluZm9CFeI/EhIQcG9zaXRpdmVFZGdlSW5mb0gAUhBwb3NpdGl2ZUVkZ2VJbmZviAEBEocBChJuZWdhd - Gl2ZV9lZGdlX2luZm8YBSABKAsyPS5zbmFwY2hhdC5yZXNlYXJjaC5nYm1sLlByZXByb2Nlc3NlZE1ldGFkYXRhLkVkZ2VNZXRhZ - GF0YUluZm9CFeI/EhIQbmVnYXRpdmVFZGdlSW5mb0gBUhBuZWdhdGl2ZUVkZ2VJbmZviAEBQhUKE19wb3NpdGl2ZV9lZGdlX2luZ - m9CFQoTX25lZ2F0aXZlX2VkZ2VfaW5mbxqxAQosQ29uZGVuc2VkTm9kZVR5cGVUb1ByZXByb2Nlc3NlZE1ldGFkYXRhRW50cnkSG - goDa2V5GAEgASgNQgjiPwUSA2tleVIDa2V5EmEKBXZhbHVlGAIgASgLMj8uc25hcGNoYXQucmVzZWFyY2guZ2JtbC5QcmVwcm9jZ - XNzZWRNZXRhZGF0YS5Ob2RlTWV0YWRhdGFPdXRwdXRCCuI/BxIFdmFsdWVSBXZhbHVlOgI4ARqxAQosQ29uZGVuc2VkRWRnZVR5c - GVUb1ByZXByb2Nlc3NlZE1ldGFkYXRhRW50cnkSGgoDa2V5GAEgASgNQgjiPwUSA2tleVIDa2V5EmEKBXZhbHVlGAIgASgLMj8uc - 25hcGNoYXQucmVzZWFyY2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5FZGdlTWV0YWRhdGFPdXRwdXRCCuI/BxIFdmFsdWVSB - XZhbHVlOgI4AWIGcHJvdG8z""" + XRzVXJpUhR0cmFuc2Zvcm1GbkFzc2V0c1VyaRKlAQoacXVhbnRpemVkX2ZlYXR1cmVfbWV0YWRhdGEYCCABKAsySC5zbmFwY2hhd + C5yZXNlYXJjaC5nYm1sLlByZXByb2Nlc3NlZE1ldGFkYXRhLkZlYXR1cmVRdWFudGl6YXRpb25NZXRhZGF0YUId4j8aEhhxdWFud + Gl6ZWRGZWF0dXJlTWV0YWRhdGFSGHF1YW50aXplZEZlYXR1cmVNZXRhZGF0YUIOCgxfZmVhdHVyZV9kaW0awgQKEkVkZ2VNZXRhZ + GF0YU91dHB1dBI4Cg9zcmNfbm9kZV9pZF9rZXkYASABKAlCEeI/DhIMc3JjTm9kZUlkS2V5UgxzcmNOb2RlSWRLZXkSOAoPZHN0X + 25vZGVfaWRfa2V5GAIgASgJQhHiPw4SDGRzdE5vZGVJZEtleVIMZHN0Tm9kZUlkS2V5EnYKDm1haW5fZWRnZV9pbmZvGAMgASgLM + j0uc25hcGNoYXQucmVzZWFyY2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5FZGdlTWV0YWRhdGFJbmZvQhHiPw4SDG1haW5FZ + GdlSW5mb1IMbWFpbkVkZ2VJbmZvEocBChJwb3NpdGl2ZV9lZGdlX2luZm8YBCABKAsyPS5zbmFwY2hhdC5yZXNlYXJjaC5nYm1sL + lByZXByb2Nlc3NlZE1ldGFkYXRhLkVkZ2VNZXRhZGF0YUluZm9CFeI/EhIQcG9zaXRpdmVFZGdlSW5mb0gAUhBwb3NpdGl2ZUVkZ + 2VJbmZviAEBEocBChJuZWdhdGl2ZV9lZGdlX2luZm8YBSABKAsyPS5zbmFwY2hhdC5yZXNlYXJjaC5nYm1sLlByZXByb2Nlc3NlZ + E1ldGFkYXRhLkVkZ2VNZXRhZGF0YUluZm9CFeI/EhIQbmVnYXRpdmVFZGdlSW5mb0gBUhBuZWdhdGl2ZUVkZ2VJbmZviAEBQhUKE + 19wb3NpdGl2ZV9lZGdlX2luZm9CFQoTX25lZ2F0aXZlX2VkZ2VfaW5mbxqxAQosQ29uZGVuc2VkTm9kZVR5cGVUb1ByZXByb2Nlc + 3NlZE1ldGFkYXRhRW50cnkSGgoDa2V5GAEgASgNQgjiPwUSA2tleVIDa2V5EmEKBXZhbHVlGAIgASgLMj8uc25hcGNoYXQucmVzZ + WFyY2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5Ob2RlTWV0YWRhdGFPdXRwdXRCCuI/BxIFdmFsdWVSBXZhbHVlOgI4ARqxA + QosQ29uZGVuc2VkRWRnZVR5cGVUb1ByZXByb2Nlc3NlZE1ldGFkYXRhRW50cnkSGgoDa2V5GAEgASgNQgjiPwUSA2tleVIDa2V5E + mEKBXZhbHVlGAIgASgLMj8uc25hcGNoYXQucmVzZWFyY2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5FZGdlTWV0YWRhdGFPd + XRwdXRCCuI/BxIFdmFsdWVSBXZhbHVlOgI4AWIGcHJvdG8z""" ).mkString) lazy val scalaDescriptor: _root_.scalapb.descriptors.FileDescriptor = { val scalaProto = com.google.protobuf.descriptor.FileDescriptorProto.parseFrom(ProtoBytes) diff --git a/scala_spark35/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadata.scala b/scala_spark35/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadata.scala index 7a9012ffa..9ddc933a7 100644 --- a/scala_spark35/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadata.scala +++ b/scala_spark35/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadata.scala @@ -1125,6 +1125,8 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r * Feature dimension after preprocessing * @param transformFnAssetsUri * Contains categorical feature vocabularies + * @param quantizedFeatureMetadata + * Optional quantized main-edge feature metadata. */ @SerialVersionUID(0L) final case class EdgeMetadataInfo( @@ -1135,6 +1137,7 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r enumeratedEdgeDataBqTable: _root_.scala.Predef.String = "", featureDim: _root_.scala.Option[_root_.scala.Int] = _root_.scala.None, transformFnAssetsUri: _root_.scala.Predef.String = "", + quantizedFeatureMetadata: _root_.scala.Option[snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata] = _root_.scala.None, unknownFields: _root_.scalapb.UnknownFieldSet = _root_.scalapb.UnknownFieldSet.empty ) extends scalapb.GeneratedMessage with scalapb.lenses.Updatable[EdgeMetadataInfo] { @transient @@ -1181,6 +1184,10 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(7, __value) } }; + if (quantizedFeatureMetadata.isDefined) { + val __value = quantizedFeatureMetadata.get + __size += 1 + _root_.com.google.protobuf.CodedOutputStream.computeUInt32SizeNoTag(__value.serializedSize) + __value.serializedSize + }; __size += unknownFields.serializedSize __size } @@ -1230,6 +1237,12 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r _output__.writeString(7, __v) } }; + quantizedFeatureMetadata.foreach { __v => + val __m = __v + _output__.writeTag(8, 2) + _output__.writeUInt32NoTag(__m.serializedSize) + __m.writeTo(_output__) + }; unknownFields.writeTo(_output__) } def clearFeatureKeys = copy(featureKeys = _root_.scala.Seq.empty) @@ -1247,6 +1260,9 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r def clearFeatureDim: EdgeMetadataInfo = copy(featureDim = _root_.scala.None) def withFeatureDim(__v: _root_.scala.Int): EdgeMetadataInfo = copy(featureDim = Option(__v)) def withTransformFnAssetsUri(__v: _root_.scala.Predef.String): EdgeMetadataInfo = copy(transformFnAssetsUri = __v) + def getQuantizedFeatureMetadata: snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata = quantizedFeatureMetadata.getOrElse(snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata.defaultInstance) + def clearQuantizedFeatureMetadata: EdgeMetadataInfo = copy(quantizedFeatureMetadata = _root_.scala.None) + def withQuantizedFeatureMetadata(__v: snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata): EdgeMetadataInfo = copy(quantizedFeatureMetadata = Option(__v)) def withUnknownFields(__v: _root_.scalapb.UnknownFieldSet) = copy(unknownFields = __v) def discardUnknownFields = copy(unknownFields = _root_.scalapb.UnknownFieldSet.empty) def getFieldByNumber(__fieldNumber: _root_.scala.Int): _root_.scala.Any = { @@ -1270,6 +1286,7 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r val __t = transformFnAssetsUri if (__t != "") __t else null } + case 8 => quantizedFeatureMetadata.orNull } } def getField(__field: _root_.scalapb.descriptors.FieldDescriptor): _root_.scalapb.descriptors.PValue = { @@ -1282,6 +1299,7 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r case 5 => _root_.scalapb.descriptors.PString(enumeratedEdgeDataBqTable) case 6 => featureDim.map(_root_.scalapb.descriptors.PInt(_)).getOrElse(_root_.scalapb.descriptors.PEmpty) case 7 => _root_.scalapb.descriptors.PString(transformFnAssetsUri) + case 8 => quantizedFeatureMetadata.map(_.toPMessage).getOrElse(_root_.scalapb.descriptors.PEmpty) } } def toProtoString: _root_.scala.Predef.String = _root_.scalapb.TextFormat.printToUnicodeString(this) @@ -1299,6 +1317,7 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r var __enumeratedEdgeDataBqTable: _root_.scala.Predef.String = "" var __featureDim: _root_.scala.Option[_root_.scala.Int] = _root_.scala.None var __transformFnAssetsUri: _root_.scala.Predef.String = "" + var __quantizedFeatureMetadata: _root_.scala.Option[snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata] = _root_.scala.None var `_unknownFields__`: _root_.scalapb.UnknownFieldSet.Builder = null var _done__ = false while (!_done__) { @@ -1319,6 +1338,8 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r __featureDim = Option(_input__.readUInt32()) case 58 => __transformFnAssetsUri = _input__.readStringRequireUtf8() + case 66 => + __quantizedFeatureMetadata = Option(__quantizedFeatureMetadata.fold(_root_.scalapb.LiteParser.readMessage[snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata](_input__))(_root_.scalapb.LiteParser.readMessage(_input__, _))) case tag => if (_unknownFields__ == null) { _unknownFields__ = new _root_.scalapb.UnknownFieldSet.Builder() @@ -1334,6 +1355,7 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r enumeratedEdgeDataBqTable = __enumeratedEdgeDataBqTable, featureDim = __featureDim, transformFnAssetsUri = __transformFnAssetsUri, + quantizedFeatureMetadata = __quantizedFeatureMetadata, unknownFields = if (_unknownFields__ == null) _root_.scalapb.UnknownFieldSet.empty else _unknownFields__.result() ) } @@ -1347,13 +1369,20 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r schemaUri = __fieldsMap.get(scalaDescriptor.findFieldByNumber(4).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), enumeratedEdgeDataBqTable = __fieldsMap.get(scalaDescriptor.findFieldByNumber(5).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), featureDim = __fieldsMap.get(scalaDescriptor.findFieldByNumber(6).get).flatMap(_.as[_root_.scala.Option[_root_.scala.Int]]), - transformFnAssetsUri = __fieldsMap.get(scalaDescriptor.findFieldByNumber(7).get).map(_.as[_root_.scala.Predef.String]).getOrElse("") + transformFnAssetsUri = __fieldsMap.get(scalaDescriptor.findFieldByNumber(7).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), + quantizedFeatureMetadata = __fieldsMap.get(scalaDescriptor.findFieldByNumber(8).get).flatMap(_.as[_root_.scala.Option[snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata]]) ) case _ => throw new RuntimeException("Expected PMessage") } def javaDescriptor: _root_.com.google.protobuf.Descriptors.Descriptor = snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.javaDescriptor.getNestedTypes().get(4) def scalaDescriptor: _root_.scalapb.descriptors.Descriptor = snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.scalaDescriptor.nestedMessages(4) - def messageCompanionForFieldNumber(__number: _root_.scala.Int): _root_.scalapb.GeneratedMessageCompanion[_] = throw new MatchError(__number) + def messageCompanionForFieldNumber(__number: _root_.scala.Int): _root_.scalapb.GeneratedMessageCompanion[_] = { + var __out: _root_.scalapb.GeneratedMessageCompanion[_] = null + (__number: @_root_.scala.unchecked) match { + case 8 => __out = snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata + } + __out + } lazy val nestedMessagesCompanions: Seq[_root_.scalapb.GeneratedMessageCompanion[_ <: _root_.scalapb.GeneratedMessage]] = Seq.empty def enumCompanionForFieldNumber(__fieldNumber: _root_.scala.Int): _root_.scalapb.GeneratedEnumCompanion[_] = throw new MatchError(__fieldNumber) lazy val defaultInstance = snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.EdgeMetadataInfo( @@ -1363,7 +1392,8 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r schemaUri = "", enumeratedEdgeDataBqTable = "", featureDim = _root_.scala.None, - transformFnAssetsUri = "" + transformFnAssetsUri = "", + quantizedFeatureMetadata = _root_.scala.None ) implicit class EdgeMetadataInfoLens[UpperPB](_l: _root_.scalapb.lenses.Lens[UpperPB, snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.EdgeMetadataInfo]) extends _root_.scalapb.lenses.ObjectLens[UpperPB, snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.EdgeMetadataInfo](_l) { def featureKeys: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Seq[_root_.scala.Predef.String]] = field(_.featureKeys)((c_, f_) => c_.copy(featureKeys = f_)) @@ -1374,6 +1404,8 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r def featureDim: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Int] = field(_.getFeatureDim)((c_, f_) => c_.copy(featureDim = Option(f_))) def optionalFeatureDim: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Option[_root_.scala.Int]] = field(_.featureDim)((c_, f_) => c_.copy(featureDim = f_)) def transformFnAssetsUri: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.transformFnAssetsUri)((c_, f_) => c_.copy(transformFnAssetsUri = f_)) + def quantizedFeatureMetadata: _root_.scalapb.lenses.Lens[UpperPB, snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata] = field(_.getQuantizedFeatureMetadata)((c_, f_) => c_.copy(quantizedFeatureMetadata = Option(f_))) + def optionalQuantizedFeatureMetadata: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Option[snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata]] = field(_.quantizedFeatureMetadata)((c_, f_) => c_.copy(quantizedFeatureMetadata = f_)) } final val FEATURE_KEYS_FIELD_NUMBER = 1 final val LABEL_KEYS_FIELD_NUMBER = 2 @@ -1382,6 +1414,7 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r final val ENUMERATED_EDGE_DATA_BQ_TABLE_FIELD_NUMBER = 5 final val FEATURE_DIM_FIELD_NUMBER = 6 final val TRANSFORM_FN_ASSETS_URI_FIELD_NUMBER = 7 + final val QUANTIZED_FEATURE_METADATA_FIELD_NUMBER = 8 def of( featureKeys: _root_.scala.Seq[_root_.scala.Predef.String], labelKeys: _root_.scala.Seq[_root_.scala.Predef.String], @@ -1389,7 +1422,8 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r schemaUri: _root_.scala.Predef.String, enumeratedEdgeDataBqTable: _root_.scala.Predef.String, featureDim: _root_.scala.Option[_root_.scala.Int], - transformFnAssetsUri: _root_.scala.Predef.String + transformFnAssetsUri: _root_.scala.Predef.String, + quantizedFeatureMetadata: _root_.scala.Option[snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata] ): _root_.snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.EdgeMetadataInfo = _root_.snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.EdgeMetadataInfo( featureKeys, labelKeys, @@ -1397,7 +1431,8 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r schemaUri, enumeratedEdgeDataBqTable, featureDim, - transformFnAssetsUri + transformFnAssetsUri, + quantizedFeatureMetadata ) // @@protoc_insertion_point(GeneratedMessageCompanion[snapchat.research.gbml.PreprocessedMetadata.EdgeMetadataInfo]) } diff --git a/scala_spark35/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadataProto.scala b/scala_spark35/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadataProto.scala index ad80de0ad..998cadd75 100644 --- a/scala_spark35/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadataProto.scala +++ b/scala_spark35/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadataProto.scala @@ -14,7 +14,7 @@ object PreprocessedMetadataProto extends _root_.scalapb.GeneratedFileObject { private lazy val ProtoBytes: _root_.scala.Array[Byte] = scalapb.Encoding.fromBase64(scala.collection.immutable.Seq( """CjJzbmFwY2hhdC9yZXNlYXJjaC9nYm1sL3ByZXByb2Nlc3NlZF9tZXRhZGF0YS5wcm90bxIWc25hcGNoYXQucmVzZWFyY2guZ - 2JtbCL9GgoUUHJlcHJvY2Vzc2VkTWV0YWRhdGES5gEKLGNvbmRlbnNlZF9ub2RlX3R5cGVfdG9fcHJlcHJvY2Vzc2VkX21ldGFkY + 2JtbCKlHAoUUHJlcHJvY2Vzc2VkTWV0YWRhdGES5gEKLGNvbmRlbnNlZF9ub2RlX3R5cGVfdG9fcHJlcHJvY2Vzc2VkX21ldGFkY XRhGAEgAygLMlkuc25hcGNoYXQucmVzZWFyY2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5Db25kZW5zZWROb2RlVHlwZVRvU HJlcHJvY2Vzc2VkTWV0YWRhdGFFbnRyeUIs4j8pEidjb25kZW5zZWROb2RlVHlwZVRvUHJlcHJvY2Vzc2VkTWV0YWRhdGFSJ2Nvb mRlbnNlZE5vZGVUeXBlVG9QcmVwcm9jZXNzZWRNZXRhZGF0YRLmAQosY29uZGVuc2VkX2VkZ2VfdHlwZV90b19wcmVwcm9jZXNzZ @@ -41,26 +41,28 @@ object PreprocessedMetadataProto extends _root_.scalapb.GeneratedFileObject { hR0cmFuc2Zvcm1GbkFzc2V0c1VyaVIUdHJhbnNmb3JtRm5Bc3NldHNVcmkSpQEKGnF1YW50aXplZF9mZWF0dXJlX21ldGFkYXRhG AogASgLMkguc25hcGNoYXQucmVzZWFyY2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5GZWF0dXJlUXVhbnRpemF0aW9uTWV0Y WRhdGFCHeI/GhIYcXVhbnRpemVkRmVhdHVyZU1ldGFkYXRhUhhxdWFudGl6ZWRGZWF0dXJlTWV0YWRhdGFCDgoMX2ZlYXR1cmVfZ - GltGugDChBFZGdlTWV0YWRhdGFJbmZvEjMKDGZlYXR1cmVfa2V5cxgBIAMoCUIQ4j8NEgtmZWF0dXJlS2V5c1ILZmVhdHVyZUtle + GltGpAFChBFZGdlTWV0YWRhdGFJbmZvEjMKDGZlYXR1cmVfa2V5cxgBIAMoCUIQ4j8NEgtmZWF0dXJlS2V5c1ILZmVhdHVyZUtle XMSLQoKbGFiZWxfa2V5cxgCIAMoCUIO4j8LEglsYWJlbEtleXNSCWxhYmVsS2V5cxJGChN0ZnJlY29yZF91cmlfcHJlZml4GAMgA SgJQhbiPxMSEXRmcmVjb3JkVXJpUHJlZml4UhF0ZnJlY29yZFVyaVByZWZpeBItCgpzY2hlbWFfdXJpGAQgASgJQg7iPwsSCXNja GVtYVVyaVIJc2NoZW1hVXJpEmAKHWVudW1lcmF0ZWRfZWRnZV9kYXRhX2JxX3RhYmxlGAUgASgJQh7iPxsSGWVudW1lcmF0ZWRFZ GdlRGF0YUJxVGFibGVSGWVudW1lcmF0ZWRFZGdlRGF0YUJxVGFibGUSNQoLZmVhdHVyZV9kaW0YBiABKA1CD+I/DBIKZmVhdHVyZ URpbUgAUgpmZWF0dXJlRGltiAEBElAKF3RyYW5zZm9ybV9mbl9hc3NldHNfdXJpGAcgASgJQhniPxYSFHRyYW5zZm9ybUZuQXNzZ - XRzVXJpUhR0cmFuc2Zvcm1GbkFzc2V0c1VyaUIOCgxfZmVhdHVyZV9kaW0awgQKEkVkZ2VNZXRhZGF0YU91dHB1dBI4Cg9zcmNfb - m9kZV9pZF9rZXkYASABKAlCEeI/DhIMc3JjTm9kZUlkS2V5UgxzcmNOb2RlSWRLZXkSOAoPZHN0X25vZGVfaWRfa2V5GAIgASgJQ - hHiPw4SDGRzdE5vZGVJZEtleVIMZHN0Tm9kZUlkS2V5EnYKDm1haW5fZWRnZV9pbmZvGAMgASgLMj0uc25hcGNoYXQucmVzZWFyY - 2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5FZGdlTWV0YWRhdGFJbmZvQhHiPw4SDG1haW5FZGdlSW5mb1IMbWFpbkVkZ2VJb - mZvEocBChJwb3NpdGl2ZV9lZGdlX2luZm8YBCABKAsyPS5zbmFwY2hhdC5yZXNlYXJjaC5nYm1sLlByZXByb2Nlc3NlZE1ldGFkY - XRhLkVkZ2VNZXRhZGF0YUluZm9CFeI/EhIQcG9zaXRpdmVFZGdlSW5mb0gAUhBwb3NpdGl2ZUVkZ2VJbmZviAEBEocBChJuZWdhd - Gl2ZV9lZGdlX2luZm8YBSABKAsyPS5zbmFwY2hhdC5yZXNlYXJjaC5nYm1sLlByZXByb2Nlc3NlZE1ldGFkYXRhLkVkZ2VNZXRhZ - GF0YUluZm9CFeI/EhIQbmVnYXRpdmVFZGdlSW5mb0gBUhBuZWdhdGl2ZUVkZ2VJbmZviAEBQhUKE19wb3NpdGl2ZV9lZGdlX2luZ - m9CFQoTX25lZ2F0aXZlX2VkZ2VfaW5mbxqxAQosQ29uZGVuc2VkTm9kZVR5cGVUb1ByZXByb2Nlc3NlZE1ldGFkYXRhRW50cnkSG - goDa2V5GAEgASgNQgjiPwUSA2tleVIDa2V5EmEKBXZhbHVlGAIgASgLMj8uc25hcGNoYXQucmVzZWFyY2guZ2JtbC5QcmVwcm9jZ - XNzZWRNZXRhZGF0YS5Ob2RlTWV0YWRhdGFPdXRwdXRCCuI/BxIFdmFsdWVSBXZhbHVlOgI4ARqxAQosQ29uZGVuc2VkRWRnZVR5c - GVUb1ByZXByb2Nlc3NlZE1ldGFkYXRhRW50cnkSGgoDa2V5GAEgASgNQgjiPwUSA2tleVIDa2V5EmEKBXZhbHVlGAIgASgLMj8uc - 25hcGNoYXQucmVzZWFyY2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5FZGdlTWV0YWRhdGFPdXRwdXRCCuI/BxIFdmFsdWVSB - XZhbHVlOgI4AWIGcHJvdG8z""" + XRzVXJpUhR0cmFuc2Zvcm1GbkFzc2V0c1VyaRKlAQoacXVhbnRpemVkX2ZlYXR1cmVfbWV0YWRhdGEYCCABKAsySC5zbmFwY2hhd + C5yZXNlYXJjaC5nYm1sLlByZXByb2Nlc3NlZE1ldGFkYXRhLkZlYXR1cmVRdWFudGl6YXRpb25NZXRhZGF0YUId4j8aEhhxdWFud + Gl6ZWRGZWF0dXJlTWV0YWRhdGFSGHF1YW50aXplZEZlYXR1cmVNZXRhZGF0YUIOCgxfZmVhdHVyZV9kaW0awgQKEkVkZ2VNZXRhZ + GF0YU91dHB1dBI4Cg9zcmNfbm9kZV9pZF9rZXkYASABKAlCEeI/DhIMc3JjTm9kZUlkS2V5UgxzcmNOb2RlSWRLZXkSOAoPZHN0X + 25vZGVfaWRfa2V5GAIgASgJQhHiPw4SDGRzdE5vZGVJZEtleVIMZHN0Tm9kZUlkS2V5EnYKDm1haW5fZWRnZV9pbmZvGAMgASgLM + j0uc25hcGNoYXQucmVzZWFyY2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5FZGdlTWV0YWRhdGFJbmZvQhHiPw4SDG1haW5FZ + GdlSW5mb1IMbWFpbkVkZ2VJbmZvEocBChJwb3NpdGl2ZV9lZGdlX2luZm8YBCABKAsyPS5zbmFwY2hhdC5yZXNlYXJjaC5nYm1sL + lByZXByb2Nlc3NlZE1ldGFkYXRhLkVkZ2VNZXRhZGF0YUluZm9CFeI/EhIQcG9zaXRpdmVFZGdlSW5mb0gAUhBwb3NpdGl2ZUVkZ + 2VJbmZviAEBEocBChJuZWdhdGl2ZV9lZGdlX2luZm8YBSABKAsyPS5zbmFwY2hhdC5yZXNlYXJjaC5nYm1sLlByZXByb2Nlc3NlZ + E1ldGFkYXRhLkVkZ2VNZXRhZGF0YUluZm9CFeI/EhIQbmVnYXRpdmVFZGdlSW5mb0gBUhBuZWdhdGl2ZUVkZ2VJbmZviAEBQhUKE + 19wb3NpdGl2ZV9lZGdlX2luZm9CFQoTX25lZ2F0aXZlX2VkZ2VfaW5mbxqxAQosQ29uZGVuc2VkTm9kZVR5cGVUb1ByZXByb2Nlc + 3NlZE1ldGFkYXRhRW50cnkSGgoDa2V5GAEgASgNQgjiPwUSA2tleVIDa2V5EmEKBXZhbHVlGAIgASgLMj8uc25hcGNoYXQucmVzZ + WFyY2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5Ob2RlTWV0YWRhdGFPdXRwdXRCCuI/BxIFdmFsdWVSBXZhbHVlOgI4ARqxA + QosQ29uZGVuc2VkRWRnZVR5cGVUb1ByZXByb2Nlc3NlZE1ldGFkYXRhRW50cnkSGgoDa2V5GAEgASgNQgjiPwUSA2tleVIDa2V5E + mEKBXZhbHVlGAIgASgLMj8uc25hcGNoYXQucmVzZWFyY2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5FZGdlTWV0YWRhdGFPd + XRwdXRCCuI/BxIFdmFsdWVSBXZhbHVlOgI4AWIGcHJvdG8z""" ).mkString) lazy val scalaDescriptor: _root_.scalapb.descriptors.FileDescriptor = { val scalaProto = com.google.protobuf.descriptor.FileDescriptorProto.parseFrom(ProtoBytes) diff --git a/snapchat/research/gbml/preprocessed_metadata_pb2.py b/snapchat/research/gbml/preprocessed_metadata_pb2.py index 2fac76d2f..5a1c46a5d 100644 --- a/snapchat/research/gbml/preprocessed_metadata_pb2.py +++ b/snapchat/research/gbml/preprocessed_metadata_pb2.py @@ -14,7 +14,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2snapchat/research/gbml/preprocessed_metadata.proto\x12\x16snapchat.research.gbml\"\x9c\x10\n\x14PreprocessedMetadata\x12\x8f\x01\n,condensed_node_type_to_preprocessed_metadata\x18\x01 \x03(\x0b\x32Y.snapchat.research.gbml.PreprocessedMetadata.CondensedNodeTypeToPreprocessedMetadataEntry\x12\x8f\x01\n,condensed_edge_type_to_preprocessed_metadata\x18\x02 \x03(\x0b\x32Y.snapchat.research.gbml.PreprocessedMetadata.CondensedEdgeTypeToPreprocessedMetadataEntry\x1aM\n\x19MultiBitQuantizationState\x12\x10\n\x08\x63lip_min\x18\x01 \x01(\x02\x12\x10\n\x08\x63lip_max\x18\x02 \x01(\x02\x12\x0c\n\x04\x62its\x18\x03 \x01(\r\x1a@\n\x1aSingleBitQuantizationState\x12\x10\n\x08neg_mean\x18\x01 \x01(\x02\x12\x10\n\x08pos_mean\x18\x02 \x01(\x02\x1a\xad\x02\n\x1b\x46\x65\x61tureQuantizationMetadata\x12\x1a\n\x12packed_feature_key\x18\x01 \x01(\t\x12!\n\x19quantized_feature_indices\x18\x02 \x03(\r\x12\x61\n\x0fmulti_bit_state\x18\x04 \x01(\x0b\x32\x46.snapchat.research.gbml.PreprocessedMetadata.MultiBitQuantizationStateH\x00\x12\x63\n\x10single_bit_state\x18\x05 \x01(\x0b\x32G.snapchat.research.gbml.PreprocessedMetadata.SingleBitQuantizationStateH\x00\x42\x07\n\x05state\x1a\x8a\x03\n\x12NodeMetadataOutput\x12\x13\n\x0bnode_id_key\x18\x01 \x01(\t\x12\x14\n\x0c\x66\x65\x61ture_keys\x18\x02 \x03(\t\x12\x12\n\nlabel_keys\x18\x03 \x03(\t\x12\x1b\n\x13tfrecord_uri_prefix\x18\x04 \x01(\t\x12\x12\n\nschema_uri\x18\x05 \x01(\t\x12$\n\x1c\x65numerated_node_ids_bq_table\x18\x06 \x01(\t\x12%\n\x1d\x65numerated_node_data_bq_table\x18\x07 \x01(\t\x12\x18\n\x0b\x66\x65\x61ture_dim\x18\x08 \x01(\rH\x00\x88\x01\x01\x12\x1f\n\x17transform_fn_assets_uri\x18\t \x01(\t\x12l\n\x1aquantized_feature_metadata\x18\n \x01(\x0b\x32H.snapchat.research.gbml.PreprocessedMetadata.FeatureQuantizationMetadataB\x0e\n\x0c_feature_dim\x1a\xdf\x01\n\x10\x45\x64geMetadataInfo\x12\x14\n\x0c\x66\x65\x61ture_keys\x18\x01 \x03(\t\x12\x12\n\nlabel_keys\x18\x02 \x03(\t\x12\x1b\n\x13tfrecord_uri_prefix\x18\x03 \x01(\t\x12\x12\n\nschema_uri\x18\x04 \x01(\t\x12%\n\x1d\x65numerated_edge_data_bq_table\x18\x05 \x01(\t\x12\x18\n\x0b\x66\x65\x61ture_dim\x18\x06 \x01(\rH\x00\x88\x01\x01\x12\x1f\n\x17transform_fn_assets_uri\x18\x07 \x01(\tB\x0e\n\x0c_feature_dim\x1a\x8b\x03\n\x12\x45\x64geMetadataOutput\x12\x17\n\x0fsrc_node_id_key\x18\x01 \x01(\t\x12\x17\n\x0f\x64st_node_id_key\x18\x02 \x01(\t\x12U\n\x0emain_edge_info\x18\x03 \x01(\x0b\x32=.snapchat.research.gbml.PreprocessedMetadata.EdgeMetadataInfo\x12^\n\x12positive_edge_info\x18\x04 \x01(\x0b\x32=.snapchat.research.gbml.PreprocessedMetadata.EdgeMetadataInfoH\x00\x88\x01\x01\x12^\n\x12negative_edge_info\x18\x05 \x01(\x0b\x32=.snapchat.research.gbml.PreprocessedMetadata.EdgeMetadataInfoH\x01\x88\x01\x01\x42\x15\n\x13_positive_edge_infoB\x15\n\x13_negative_edge_info\x1a\x8f\x01\n,CondensedNodeTypeToPreprocessedMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\r\x12N\n\x05value\x18\x02 \x01(\x0b\x32?.snapchat.research.gbml.PreprocessedMetadata.NodeMetadataOutput:\x02\x38\x01\x1a\x8f\x01\n,CondensedEdgeTypeToPreprocessedMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\r\x12N\n\x05value\x18\x02 \x01(\x0b\x32?.snapchat.research.gbml.PreprocessedMetadata.EdgeMetadataOutput:\x02\x38\x01\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2snapchat/research/gbml/preprocessed_metadata.proto\x12\x16snapchat.research.gbml\"\x8a\x11\n\x14PreprocessedMetadata\x12\x8f\x01\n,condensed_node_type_to_preprocessed_metadata\x18\x01 \x03(\x0b\x32Y.snapchat.research.gbml.PreprocessedMetadata.CondensedNodeTypeToPreprocessedMetadataEntry\x12\x8f\x01\n,condensed_edge_type_to_preprocessed_metadata\x18\x02 \x03(\x0b\x32Y.snapchat.research.gbml.PreprocessedMetadata.CondensedEdgeTypeToPreprocessedMetadataEntry\x1aM\n\x19MultiBitQuantizationState\x12\x10\n\x08\x63lip_min\x18\x01 \x01(\x02\x12\x10\n\x08\x63lip_max\x18\x02 \x01(\x02\x12\x0c\n\x04\x62its\x18\x03 \x01(\r\x1a@\n\x1aSingleBitQuantizationState\x12\x10\n\x08neg_mean\x18\x01 \x01(\x02\x12\x10\n\x08pos_mean\x18\x02 \x01(\x02\x1a\xad\x02\n\x1b\x46\x65\x61tureQuantizationMetadata\x12\x1a\n\x12packed_feature_key\x18\x01 \x01(\t\x12!\n\x19quantized_feature_indices\x18\x02 \x03(\r\x12\x61\n\x0fmulti_bit_state\x18\x04 \x01(\x0b\x32\x46.snapchat.research.gbml.PreprocessedMetadata.MultiBitQuantizationStateH\x00\x12\x63\n\x10single_bit_state\x18\x05 \x01(\x0b\x32G.snapchat.research.gbml.PreprocessedMetadata.SingleBitQuantizationStateH\x00\x42\x07\n\x05state\x1a\x8a\x03\n\x12NodeMetadataOutput\x12\x13\n\x0bnode_id_key\x18\x01 \x01(\t\x12\x14\n\x0c\x66\x65\x61ture_keys\x18\x02 \x03(\t\x12\x12\n\nlabel_keys\x18\x03 \x03(\t\x12\x1b\n\x13tfrecord_uri_prefix\x18\x04 \x01(\t\x12\x12\n\nschema_uri\x18\x05 \x01(\t\x12$\n\x1c\x65numerated_node_ids_bq_table\x18\x06 \x01(\t\x12%\n\x1d\x65numerated_node_data_bq_table\x18\x07 \x01(\t\x12\x18\n\x0b\x66\x65\x61ture_dim\x18\x08 \x01(\rH\x00\x88\x01\x01\x12\x1f\n\x17transform_fn_assets_uri\x18\t \x01(\t\x12l\n\x1aquantized_feature_metadata\x18\n \x01(\x0b\x32H.snapchat.research.gbml.PreprocessedMetadata.FeatureQuantizationMetadataB\x0e\n\x0c_feature_dim\x1a\xcd\x02\n\x10\x45\x64geMetadataInfo\x12\x14\n\x0c\x66\x65\x61ture_keys\x18\x01 \x03(\t\x12\x12\n\nlabel_keys\x18\x02 \x03(\t\x12\x1b\n\x13tfrecord_uri_prefix\x18\x03 \x01(\t\x12\x12\n\nschema_uri\x18\x04 \x01(\t\x12%\n\x1d\x65numerated_edge_data_bq_table\x18\x05 \x01(\t\x12\x18\n\x0b\x66\x65\x61ture_dim\x18\x06 \x01(\rH\x00\x88\x01\x01\x12\x1f\n\x17transform_fn_assets_uri\x18\x07 \x01(\t\x12l\n\x1aquantized_feature_metadata\x18\x08 \x01(\x0b\x32H.snapchat.research.gbml.PreprocessedMetadata.FeatureQuantizationMetadataB\x0e\n\x0c_feature_dim\x1a\x8b\x03\n\x12\x45\x64geMetadataOutput\x12\x17\n\x0fsrc_node_id_key\x18\x01 \x01(\t\x12\x17\n\x0f\x64st_node_id_key\x18\x02 \x01(\t\x12U\n\x0emain_edge_info\x18\x03 \x01(\x0b\x32=.snapchat.research.gbml.PreprocessedMetadata.EdgeMetadataInfo\x12^\n\x12positive_edge_info\x18\x04 \x01(\x0b\x32=.snapchat.research.gbml.PreprocessedMetadata.EdgeMetadataInfoH\x00\x88\x01\x01\x12^\n\x12negative_edge_info\x18\x05 \x01(\x0b\x32=.snapchat.research.gbml.PreprocessedMetadata.EdgeMetadataInfoH\x01\x88\x01\x01\x42\x15\n\x13_positive_edge_infoB\x15\n\x13_negative_edge_info\x1a\x8f\x01\n,CondensedNodeTypeToPreprocessedMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\r\x12N\n\x05value\x18\x02 \x01(\x0b\x32?.snapchat.research.gbml.PreprocessedMetadata.NodeMetadataOutput:\x02\x38\x01\x1a\x8f\x01\n,CondensedEdgeTypeToPreprocessedMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\r\x12N\n\x05value\x18\x02 \x01(\x0b\x32?.snapchat.research.gbml.PreprocessedMetadata.EdgeMetadataOutput:\x02\x38\x01\x62\x06proto3') @@ -106,7 +106,7 @@ _PREPROCESSEDMETADATA_CONDENSEDEDGETYPETOPREPROCESSEDMETADATAENTRY._options = None _PREPROCESSEDMETADATA_CONDENSEDEDGETYPETOPREPROCESSEDMETADATAENTRY._serialized_options = b'8\001' _PREPROCESSEDMETADATA._serialized_start=79 - _PREPROCESSEDMETADATA._serialized_end=2155 + _PREPROCESSEDMETADATA._serialized_end=2265 _PREPROCESSEDMETADATA_MULTIBITQUANTIZATIONSTATE._serialized_start=395 _PREPROCESSEDMETADATA_MULTIBITQUANTIZATIONSTATE._serialized_end=472 _PREPROCESSEDMETADATA_SINGLEBITQUANTIZATIONSTATE._serialized_start=474 @@ -116,11 +116,11 @@ _PREPROCESSEDMETADATA_NODEMETADATAOUTPUT._serialized_start=845 _PREPROCESSEDMETADATA_NODEMETADATAOUTPUT._serialized_end=1239 _PREPROCESSEDMETADATA_EDGEMETADATAINFO._serialized_start=1242 - _PREPROCESSEDMETADATA_EDGEMETADATAINFO._serialized_end=1465 - _PREPROCESSEDMETADATA_EDGEMETADATAOUTPUT._serialized_start=1468 - _PREPROCESSEDMETADATA_EDGEMETADATAOUTPUT._serialized_end=1863 - _PREPROCESSEDMETADATA_CONDENSEDNODETYPETOPREPROCESSEDMETADATAENTRY._serialized_start=1866 - _PREPROCESSEDMETADATA_CONDENSEDNODETYPETOPREPROCESSEDMETADATAENTRY._serialized_end=2009 - _PREPROCESSEDMETADATA_CONDENSEDEDGETYPETOPREPROCESSEDMETADATAENTRY._serialized_start=2012 - _PREPROCESSEDMETADATA_CONDENSEDEDGETYPETOPREPROCESSEDMETADATAENTRY._serialized_end=2155 + _PREPROCESSEDMETADATA_EDGEMETADATAINFO._serialized_end=1575 + _PREPROCESSEDMETADATA_EDGEMETADATAOUTPUT._serialized_start=1578 + _PREPROCESSEDMETADATA_EDGEMETADATAOUTPUT._serialized_end=1973 + _PREPROCESSEDMETADATA_CONDENSEDNODETYPETOPREPROCESSEDMETADATAENTRY._serialized_start=1976 + _PREPROCESSEDMETADATA_CONDENSEDNODETYPETOPREPROCESSEDMETADATAENTRY._serialized_end=2119 + _PREPROCESSEDMETADATA_CONDENSEDEDGETYPETOPREPROCESSEDMETADATAENTRY._serialized_start=2122 + _PREPROCESSEDMETADATA_CONDENSEDEDGETYPETOPREPROCESSEDMETADATAENTRY._serialized_end=2265 # @@protoc_insertion_point(module_scope) diff --git a/snapchat/research/gbml/preprocessed_metadata_pb2.pyi b/snapchat/research/gbml/preprocessed_metadata_pb2.pyi index 46b80c7bb..8ae9c79a5 100644 --- a/snapchat/research/gbml/preprocessed_metadata_pb2.pyi +++ b/snapchat/research/gbml/preprocessed_metadata_pb2.pyi @@ -154,6 +154,7 @@ class PreprocessedMetadata(google.protobuf.message.Message): ENUMERATED_EDGE_DATA_BQ_TABLE_FIELD_NUMBER: builtins.int FEATURE_DIM_FIELD_NUMBER: builtins.int TRANSFORM_FN_ASSETS_URI_FIELD_NUMBER: builtins.int + QUANTIZED_FEATURE_METADATA_FIELD_NUMBER: builtins.int @property def feature_keys(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: """Fields in output TFRecords which reference features.""" @@ -170,6 +171,9 @@ class PreprocessedMetadata(google.protobuf.message.Message): """Feature dimension after preprocessing""" transform_fn_assets_uri: builtins.str """Contains categorical feature vocabularies""" + @property + def quantized_feature_metadata(self) -> global___PreprocessedMetadata.FeatureQuantizationMetadata: + """Optional quantized main-edge feature metadata.""" def __init__( self, *, @@ -180,9 +184,10 @@ class PreprocessedMetadata(google.protobuf.message.Message): enumerated_edge_data_bq_table: builtins.str = ..., feature_dim: builtins.int | None = ..., transform_fn_assets_uri: builtins.str = ..., + quantized_feature_metadata: global___PreprocessedMetadata.FeatureQuantizationMetadata | None = ..., ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["_feature_dim", b"_feature_dim", "feature_dim", b"feature_dim"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["_feature_dim", b"_feature_dim", "enumerated_edge_data_bq_table", b"enumerated_edge_data_bq_table", "feature_dim", b"feature_dim", "feature_keys", b"feature_keys", "label_keys", b"label_keys", "schema_uri", b"schema_uri", "tfrecord_uri_prefix", b"tfrecord_uri_prefix", "transform_fn_assets_uri", b"transform_fn_assets_uri"]) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_feature_dim", b"_feature_dim", "feature_dim", b"feature_dim", "quantized_feature_metadata", b"quantized_feature_metadata"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_feature_dim", b"_feature_dim", "enumerated_edge_data_bq_table", b"enumerated_edge_data_bq_table", "feature_dim", b"feature_dim", "feature_keys", b"feature_keys", "label_keys", b"label_keys", "quantized_feature_metadata", b"quantized_feature_metadata", "schema_uri", b"schema_uri", "tfrecord_uri_prefix", b"tfrecord_uri_prefix", "transform_fn_assets_uri", b"transform_fn_assets_uri"]) -> None: ... def WhichOneof(self, oneof_group: typing_extensions.Literal["_feature_dim", b"_feature_dim"]) -> typing_extensions.Literal["feature_dim"] | None: ... class EdgeMetadataOutput(google.protobuf.message.Message):