QVAC-22254 feat: isolated LAM-A2E C API and GGML graph

.github/workflows/build.yml

@@ -178,6 +178,44 @@ jobs:
           path: |
             sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-vulkan.zip
 
+  ubuntu-latest-lam-a2e:
+    # Compile-checks the isolated LAM audio2expression C API and its smoke /
+    # CPU-parity harness (only built under -DSD_BUILD_LAM_A2E_SMOKE=ON, so the
+    # standard build jobs above don't cover the test targets). This guards
+    # against regressions in include/lam-a2e.h, src/lam_audio2expression.* and
+    # tests/lam_a2e_*.cpp.
+    #
+    # NOTE: this job only *builds* the harness. Running lam-a2e-smoke /
+    # lam-a2e-parity needs the remapped lam-audio2exp GGUF plus the
+    # lipsync-ggml reference fixtures, neither of which is available in CI yet.
+    # Wire the execution step in once those assets are hosted (tracked by
+    # QVAC-22250 / QVAC-22249).
+    #
+    # The GGUF conversion and fixture-dump tooling lives with the consumer, in
+    # qvac packages/diffusion-cpp/scripts/ (see README-lam-a2e.md there).
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Dependencies
+        id: depends
+        run: |
+          sudo apt-get update
+          sudo apt-get install build-essential
+
+      - name: Build LAM-A2E smoke + parity targets
+        id: cmake_build
+        run: |
+          mkdir build
+          cd build
+          cmake .. -DGGML_AVX2=ON -DSD_BUILD_EXAMPLES=OFF -DSD_BUILD_LAM_A2E_SMOKE=ON
+          cmake --build . --config Release --target lam-a2e-smoke lam-a2e-frontend-smoke lam-a2e-parity
+
   build-and-push-docker-images:
     name: Build and push container images
     if: ${{ github.event_name != 'pull_request' }}

CMakeLists.txt

@@ -273,6 +273,8 @@ else()
     add_library(${SD_LIB} STATIC ${SD_LIB_SOURCES})
 endif()
 
+option(SD_BUILD_LAM_A2E_SMOKE "Build the LAM-A2E GGUF loader smoke executable" OFF)
+
 if(APPLE)
     sd_set_macos_rpaths(${SD_LIB})
 endif()
@@ -335,12 +337,29 @@ target_include_directories(${SD_LIB} PUBLIC . src include)
 target_include_directories(${SD_LIB} PUBLIC . thirdparty)
 target_compile_features(${SD_LIB} PUBLIC c_std_11 cxx_std_17)
 
+if(SD_BUILD_LAM_A2E_SMOKE)
+    add_executable(lam-a2e-smoke tests/lam_a2e_smoke.cpp)
+    target_include_directories(lam-a2e-smoke PRIVATE include)
+    target_link_libraries(lam-a2e-smoke PRIVATE ${SD_LIB})
+    target_compile_features(lam-a2e-smoke PRIVATE cxx_std_17)
+
+    add_executable(lam-a2e-frontend-smoke tests/lam_a2e_frontend_smoke.cpp)
+    target_include_directories(lam-a2e-frontend-smoke PRIVATE include src)
+    target_link_libraries(lam-a2e-frontend-smoke PRIVATE ${SD_LIB})
+    target_compile_features(lam-a2e-frontend-smoke PRIVATE cxx_std_17)
+
+    add_executable(lam-a2e-parity tests/lam_a2e_parity.cpp)
+    target_include_directories(lam-a2e-parity PRIVATE include src)
+    target_link_libraries(lam-a2e-parity PRIVATE ${SD_LIB})
+    target_compile_features(lam-a2e-parity PRIVATE cxx_std_17)
+endif()
+
 
 if (SD_BUILD_EXAMPLES)
     add_subdirectory(examples)
 endif()
 
-set(SD_PUBLIC_HEADERS include/stable-diffusion.h)
+set(SD_PUBLIC_HEADERS include/stable-diffusion.h include/lam-a2e.h)
 set_target_properties(${SD_LIB} PROPERTIES PUBLIC_HEADER "${SD_PUBLIC_HEADERS}")
 
 install(TARGETS ${SD_LIB} LIBRARY PUBLIC_HEADER)

include/lam-a2e.h

@@ -0,0 +1,47 @@
+#pragma once
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct lam_a2e_context lam_a2e_context;
+
+typedef enum lam_a2e_status {
+    LAM_A2E_STATUS_OK = 0,
+    LAM_A2E_STATUS_INVALID_ARGUMENT = 1,
+    LAM_A2E_STATUS_MODEL_LOAD_FAILED = 2,
+    LAM_A2E_STATUS_NOT_IMPLEMENTED = 3,
+} lam_a2e_status;
+
+typedef struct lam_a2e_params {
+    const char * model_path;
+    int32_t identity_index;
+    int32_t n_threads;
+    bool use_gpu;
+} lam_a2e_params;
+
+typedef struct lam_a2e_frame {
+    int64_t timestamp_us;
+    float arkit_52[52];
+} lam_a2e_frame;
+
+lam_a2e_context * lam_a2e_create(const lam_a2e_params * params);
+void lam_a2e_free(lam_a2e_context * ctx);
+
+lam_a2e_status lam_a2e_process_pcm_f32(
+    lam_a2e_context * ctx,
+    const float * pcm,
+    int64_t pcm_sample_count,
+    int32_t sample_rate,
+    lam_a2e_frame ** frames,
+    int32_t * frame_count);
+
+void lam_a2e_free_frames(lam_a2e_frame * frames);
+const char * lam_a2e_get_last_error(const lam_a2e_context * ctx);
+
+#ifdef __cplusplus
+}
+#endif

src/lam-a2e.cpp

@@ -0,0 +1,120 @@
+#include "lam-a2e.h"
+
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "lam_audio2expression.hpp"
+
+struct lam_a2e_context {
+    std::string last_error;
+    std::unique_ptr<LamAudio2Expression> model;
+    int32_t identity_index = 0;
+    int32_t n_threads = 0;
+    bool use_gpu = false;
+    bool model_loaded = false;
+};
+
+lam_a2e_context * lam_a2e_create(const lam_a2e_params * params) {
+    if (params == nullptr || params->model_path == nullptr ||
+        std::strlen(params->model_path) == 0) {
+        return nullptr;
+    }
+
+    auto * ctx = new lam_a2e_context();
+    ctx->identity_index = params->identity_index;
+    ctx->n_threads = params->n_threads;
+    ctx->use_gpu = params->use_gpu;
+
+    if (params->use_gpu) {
+        ctx->last_error =
+            "LAM-A2E GPU backends are not enabled until CPU parity is complete.";
+        return ctx;
+    }
+
+    ctx->model = std::make_unique<LamAudio2Expression>();
+    if (!ctx->model->load(params->model_path, nullptr, params->n_threads)) {
+        ctx->last_error = ctx->model->lastError();
+        ctx->model.reset();
+        return ctx;
+    }
+
+    ctx->model_loaded = true;
+    ctx->last_error.clear();
+    return ctx;
+}
+
+void lam_a2e_free(lam_a2e_context * ctx) {
+    delete ctx;
+}
+
+lam_a2e_status lam_a2e_process_pcm_f32(
+    lam_a2e_context * ctx,
+    const float * pcm,
+    int64_t pcm_sample_count,
+    int32_t sample_rate,
+    lam_a2e_frame ** frames,
+    int32_t * frame_count) {
+    if (ctx == nullptr || pcm == nullptr || pcm_sample_count <= 0 ||
+        frames == nullptr || frame_count == nullptr) {
+        return LAM_A2E_STATUS_INVALID_ARGUMENT;
+    }
+
+    *frames = nullptr;
+    *frame_count = 0;
+
+    if (!ctx->model_loaded || ctx->model == nullptr) {
+        if (ctx->last_error.empty()) {
+            ctx->last_error = "LAM-A2E model is not loaded.";
+        }
+        return LAM_A2E_STATUS_MODEL_LOAD_FAILED;
+    }
+    if (sample_rate != 16000) {
+        ctx->last_error = "LAM-A2E requires 16 kHz mono PCM input.";
+        return LAM_A2E_STATUS_INVALID_ARGUMENT;
+    }
+
+    std::vector<float> pcm_vec(pcm, pcm + pcm_sample_count);
+    std::vector<float> coeffs;
+    if (!ctx->model->run(pcm_vec, static_cast<uint32_t>(ctx->identity_index), coeffs)) {
+        ctx->last_error = ctx->model->lastError();
+        return LAM_A2E_STATUS_INVALID_ARGUMENT;
+    }
+
+    const auto& hp = ctx->model->hparams();
+    const int64_t n_frames = ctx->model->frameCount(pcm_sample_count);
+    if (n_frames <= 0 ||
+        coeffs.size() != static_cast<size_t>(n_frames) * hp.nCoeffs) {
+        ctx->last_error = "LAM-A2E produced an unexpected coefficient buffer size.";
+        return LAM_A2E_STATUS_INVALID_ARGUMENT;
+    }
+
+    auto * out = static_cast<lam_a2e_frame *>(
+        std::malloc(sizeof(lam_a2e_frame) * static_cast<size_t>(n_frames)));
+    if (out == nullptr) {
+        ctx->last_error = "Failed to allocate LAM-A2E frame buffer.";
+        return LAM_A2E_STATUS_INVALID_ARGUMENT;
+    }
+
+    for (int64_t i = 0; i < n_frames; ++i) {
+        out[i].timestamp_us =
+            (i * 1000000LL) / static_cast<int64_t>(hp.fps > 0 ? hp.fps : 30);
+        std::memcpy(out[i].arkit_52, coeffs.data() + i * hp.nCoeffs,
+                    sizeof(float) * hp.nCoeffs);
+    }
+
+    *frames = out;
+    *frame_count = static_cast<int32_t>(n_frames);
+    ctx->last_error.clear();
+    return LAM_A2E_STATUS_OK;
+}
+
+void lam_a2e_free_frames(lam_a2e_frame * frames) {
+    std::free(frames);
+}
+
+const char * lam_a2e_get_last_error(const lam_a2e_context * ctx) {
+    return ctx == nullptr ? "LAM-A2E context is null." : ctx->last_error.c_str();
+}