From d723f53cba977ee7e55731d050eff931df5dd177 Mon Sep 17 00:00:00 2001 From: Aaron Po Date: Sun, 19 Jul 2026 04:10:33 -0400 Subject: [PATCH] Formatted src/data_generation/llama --- .../data_generation/llama/generate_brewery.cc | 236 ++++++------ .../data_generation/llama/generate_user.cc | 166 ++++----- .../src/data_generation/llama/helpers.cc | 329 ++++++++--------- .../src/data_generation/llama/infer.cc | 342 +++++++++--------- .../data_generation/llama/llama_generator.cc | 102 +++--- .../src/data_generation/llama/load.cc | 64 ++-- 6 files changed, 622 insertions(+), 617 deletions(-) diff --git a/tooling/pipeline/src/data_generation/llama/generate_brewery.cc b/tooling/pipeline/src/data_generation/llama/generate_brewery.cc index f9362bd2..d110e5f3 100644 --- a/tooling/pipeline/src/data_generation/llama/generate_brewery.cc +++ b/tooling/pipeline/src/data_generation/llama/generate_brewery.cc @@ -18,144 +18,144 @@ static std::string FormatLocalLanguageCodes( const std::vector& codes) { - if (codes.empty()) { - return "Not provided"; - } + if (codes.empty()) { + return "Not provided"; + } - std::string formatted; - for (const std::string& code : codes) { - if (!formatted.empty()) { - formatted += ", "; - } - formatted += code; - } + std::string formatted; + for (const std::string& code : codes) { + if (!formatted.empty()) { + formatted += ", "; + } + formatted += code; + } - return formatted; + return formatted; } static constexpr int kBreweryInitialMaxTokens = 2800; BreweryResult LlamaGenerator::GenerateBrewery( const EnrichedCity& enriched_city) { - const City& location = enriched_city.location; - const std::string& region_context = enriched_city.region_context; - /** - * Preprocess and truncate region context to manageable size - */ - const std::string safe_region_context = PrepareRegionContext(region_context); + const City& location = enriched_city.location; + const std::string& region_context = enriched_city.region_context; + /** + * Preprocess and truncate region context to manageable size + */ + const std::string safe_region_context = PrepareRegionContext(region_context); - const std::string local_language_codes = - FormatLocalLanguageCodes(location.local_languages); + const std::string local_language_codes = + FormatLocalLanguageCodes(location.local_languages); - const std::string country_suffix = - location.country.empty() ? std::string{} - : std::format(", {}", location.country); - /** - * Load brewery system prompt via the injected prompt directory. - * The key "BREWERY_GENERATION" resolves to BREWERY_GENERATION.md inside - * the configured --prompt-dir. Throws on missing or empty file. - */ - const std::string system_prompt = - prompt_directory_->Load("BREWERY_GENERATION"); + const std::string country_suffix = + location.country.empty() ? std::string{} + : std::format(", {}", location.country); + /** + * Load brewery system prompt via the injected prompt directory. + * The key "BREWERY_GENERATION" resolves to BREWERY_GENERATION.md inside + * the configured --prompt-dir. Throws on missing or empty file. + */ + const std::string system_prompt = + prompt_directory_->Load("BREWERY_GENERATION"); - std::string user_prompt = std::format( - "## CITY:\n{}\n\n## COUNTRY:\n{}\n\n## LOCAL LANGUAGE CODES:\n{}\n\n## " - "CONTEXT:\n{}", - location.city, location.country, local_language_codes, - safe_region_context); + std::string user_prompt = std::format( + "## CITY:\n{}\n\n## COUNTRY:\n{}\n\n## LOCAL LANGUAGE CODES:\n{}\n\n## " + "CONTEXT:\n{}", + location.city, location.country, local_language_codes, + safe_region_context); - /** - * Store location context for retry prompts (without repeating full context) - */ - const std::string retry_location = - std::format("City: {}{}\nLocal language codes: {}", location.city, - country_suffix, local_language_codes); + /** + * Store location context for retry prompts (without repeating full context) + */ + const std::string retry_location = + std::format("City: {}{}\nLocal language codes: {}", location.city, + country_suffix, local_language_codes); - /** - * RETRY LOOP with validation and error correction - * Attempts to generate valid brewery data up to 3 times, with - * feedback-based refinement - */ - constexpr int max_attempts = 3; - std::string raw; - std::string last_error; + /** + * RETRY LOOP with validation and error correction + * Attempts to generate valid brewery data up to 3 times, with + * feedback-based refinement + */ + constexpr int max_attempts = 3; + std::string raw; + std::string last_error; - // Token budget: too small risks truncating valid JSON mid-string. - int max_tokens = kBreweryInitialMaxTokens; - - // Limit output length to keep it concise and focused - for (int attempt = 0; attempt < max_attempts; ++attempt) { - // Generate brewery data from LLM - raw = this->Infer(system_prompt, user_prompt, max_tokens, - kBreweryJsonGrammar); - if (logger_) { - logger_->Log( - {.level = LogLevel::Debug, - .phase = PipelinePhase::BreweryAndBeerGeneration, - .message = std::format("LlamaGenerator: raw output (attempt {}): {}", - attempt + 1, raw)}); - } - - // Validate output: parse JSON and check required fields - - BreweryResult brewery; - const std::optional validation_error = - ValidateBreweryJson(raw, brewery); - - if (!validation_error.has_value()) { - // Success: return parsed brewery data + // Token budget: too small risks truncating valid JSON mid-string. + int max_tokens = kBreweryInitialMaxTokens; + // Limit output length to keep it concise and focused + for (int attempt = 0; attempt < max_attempts; ++attempt) { + // Generate brewery data from LLM + raw = this->Infer(system_prompt, user_prompt, max_tokens, + kBreweryJsonGrammar); if (logger_) { - logger_->Log( - {.level = LogLevel::Info, - .phase = PipelinePhase::BreweryAndBeerGeneration, - .message = std::format("LlamaGenerator: successfully generated " - "brewery data on attempt {}", - attempt + 1)}); + logger_->Log({.level = LogLevel::Debug, + .phase = PipelinePhase::BreweryAndBeerGeneration, + .message = std::format( + "LlamaGenerator: raw output (attempt {}): {}", + attempt + 1, raw)}); } - return brewery; - } + // Validate output: parse JSON and check required fields - // Validation failed: log error and prepare corrective feedback + BreweryResult brewery; + const std::optional validation_error = + ValidateBreweryJson(raw, brewery); - last_error = *validation_error; - if (logger_) { - logger_->Log( - {.level = LogLevel::Warn, - .phase = PipelinePhase::BreweryAndBeerGeneration, - .message = std::format( - "LlamaGenerator: malformed brewery JSON (attempt {}): {}", - attempt + 1, *validation_error)}); - } + if (!validation_error.has_value()) { + // Success: return parsed brewery data - // Update prompt with error details to guide LLM toward correct output. - user_prompt = std::format( - "Your previous response was invalid. Error: {}\nReturn the thought " - "process before the JSON if needed, then return ONLY valid JSON " - "with " - "exactly these keys, in this exact order: {{\"name_en\": " - "\"\", \"description_en\": \"\", \"name_local\": \"\", " - "\"description_local\": \"\"}}.\nDo not include markdown, comments, extra keys, " - "or " - "literal placeholder values.\n\nKeep the JSON strings concise " - "enough " - "to fit within the token budget.\n\n{}", - *validation_error, retry_location); - } + if (logger_) { + logger_->Log({.level = LogLevel::Info, + .phase = PipelinePhase::BreweryAndBeerGeneration, + .message = std::format( + "LlamaGenerator: successfully generated " + "brewery data on attempt {}", + attempt + 1)}); + } - // All retry attempts exhausted: log failure and throw exception - if (logger_) { - logger_->Log({.level = LogLevel::Error, - .phase = PipelinePhase::BreweryAndBeerGeneration, - .message = std::format( - "LlamaGenerator: malformed brewery " - "response after {} attempts: {}", - max_attempts, last_error.empty() ? raw : last_error)}); - } - throw std::runtime_error("LlamaGenerator: malformed brewery response"); + return brewery; + } + + // Validation failed: log error and prepare corrective feedback + + last_error = *validation_error; + if (logger_) { + logger_->Log( + {.level = LogLevel::Warn, + .phase = PipelinePhase::BreweryAndBeerGeneration, + .message = std::format( + "LlamaGenerator: malformed brewery JSON (attempt {}): {}", + attempt + 1, *validation_error)}); + } + + // Update prompt with error details to guide LLM toward correct output. + user_prompt = std::format( + "Your previous response was invalid. Error: {}\nReturn the thought " + "process before the JSON if needed, then return ONLY valid JSON " + "with " + "exactly these keys, in this exact order: {{\"name_en\": " + "\"\", \"description_en\": \"\", \"name_local\": \"\", " + "\"description_local\": \"\"}}.\nDo not include markdown, comments, extra keys, " + "or " + "literal placeholder values.\n\nKeep the JSON strings concise " + "enough " + "to fit within the token budget.\n\n{}", + *validation_error, retry_location); + } + + // All retry attempts exhausted: log failure and throw exception + if (logger_) { + logger_->Log({.level = LogLevel::Error, + .phase = PipelinePhase::BreweryAndBeerGeneration, + .message = std::format( + "LlamaGenerator: malformed brewery " + "response after {} attempts: {}", + max_attempts, last_error.empty() ? raw : last_error)}); + } + throw std::runtime_error("LlamaGenerator: malformed brewery response"); } diff --git a/tooling/pipeline/src/data_generation/llama/generate_user.cc b/tooling/pipeline/src/data_generation/llama/generate_user.cc index cb359008..8eb68c6b 100644 --- a/tooling/pipeline/src/data_generation/llama/generate_user.cc +++ b/tooling/pipeline/src/data_generation/llama/generate_user.cc @@ -19,97 +19,99 @@ static constexpr int kUserInitialMaxTokens = 1200; UserResult LlamaGenerator::GenerateUser(const EnrichedCity& city, const UserPersona& persona, const Name& name) { - std::string style_affinities; - for (const std::string& style : persona.style_affinities) { - if (!style_affinities.empty()) { - style_affinities += ", "; - } - style_affinities += style; - } + std::string style_affinities; + for (const std::string& style : persona.style_affinities) { + if (!style_affinities.empty()) { + style_affinities += ", "; + } + style_affinities += style; + } - const std::string system_prompt = prompt_directory_->Load("USER_GENERATION"); + const std::string system_prompt = prompt_directory_->Load("USER_GENERATION"); - std::string user_prompt = std::format( - "## NAME:\n\n{} {}\n\n" - "## GENDER:\n\n{}\n\n" - "## CITY:\n\n{}\n\n" - "## COUNTRY:\n\n{}\n\n" - "## PERSONA:\n\n{}\n\n" - "## PERSONA DESCRIPTION:\n\n{}\n\n" - "## STYLE AFFINITIES:\n\n{}", - name.first_name, name.last_name, name.gender, city.location.city, - city.location.country, persona.name, persona.description, - style_affinities); + std::string user_prompt = std::format( + "## NAME:\n\n{} {}\n\n" + "## GENDER:\n\n{}\n\n" + "## CITY:\n\n{}\n\n" + "## COUNTRY:\n\n{}\n\n" + "## PERSONA:\n\n{}\n\n" + "## PERSONA DESCRIPTION:\n\n{}\n\n" + "## STYLE AFFINITIES:\n\n{}", + name.first_name, name.last_name, name.gender, city.location.city, + city.location.country, persona.name, persona.description, + style_affinities); - const std::string retry_context = std::format( - "Name: {} {}\nCity: {}, {}\nPersona: {}", name.first_name, name.last_name, - city.location.city, city.location.country, persona.name); + const std::string retry_context = std::format( + "Name: {} {}\nCity: {}, {}\nPersona: {}", name.first_name, + name.last_name, city.location.city, city.location.country, persona.name); - constexpr int max_attempts = 3; - std::string raw; - std::string last_error; - int max_tokens = kUserInitialMaxTokens; + constexpr int max_attempts = 3; + std::string raw; + std::string last_error; + int max_tokens = kUserInitialMaxTokens; - for (int attempt = 0; attempt < max_attempts; ++attempt) { - raw = this->Infer(system_prompt, user_prompt, max_tokens, kUserJsonGrammar); - if (logger_) { - logger_->Log( - {.level = LogLevel::Debug, - .phase = PipelinePhase::UserGeneration, - .message = std::format("LlamaGenerator: raw output (attempt {}): {}", - attempt + 1, raw)}); - } - - UserResult user; - const std::optional validation_error = - ValidateUserJson(raw, user); - - if (!validation_error.has_value()) { + for (int attempt = 0; attempt < max_attempts; ++attempt) { + raw = + this->Infer(system_prompt, user_prompt, max_tokens, kUserJsonGrammar); if (logger_) { - logger_->Log( - {.level = LogLevel::Info, - .phase = PipelinePhase::UserGeneration, - .message = std::format("LlamaGenerator: successfully " - "generated user data on attempt {}", - attempt + 1)}); + logger_->Log({.level = LogLevel::Debug, + .phase = PipelinePhase::UserGeneration, + .message = std::format( + "LlamaGenerator: raw output (attempt {}): {}", + attempt + 1, raw)}); } - user.first_name = name.first_name; - user.last_name = name.last_name; - user.gender = name.gender; - return user; - } + UserResult user; + const std::optional validation_error = + ValidateUserJson(raw, user); - last_error = *validation_error; - if (logger_) { - logger_->Log({.level = LogLevel::Warn, + if (!validation_error.has_value()) { + if (logger_) { + logger_->Log( + {.level = LogLevel::Info, + .phase = PipelinePhase::UserGeneration, + .message = std::format("LlamaGenerator: successfully " + "generated user data on attempt {}", + attempt + 1)}); + } + + user.first_name = name.first_name; + user.last_name = name.last_name; + user.gender = name.gender; + return user; + } + + last_error = *validation_error; + if (logger_) { + logger_->Log( + {.level = LogLevel::Warn, + .phase = PipelinePhase::UserGeneration, + .message = std::format( + "LlamaGenerator: malformed user JSON (attempt {}): {}", + attempt + 1, *validation_error)}); + } + + user_prompt = std::format( + "Your previous response was invalid. Error: {}\nReturn the thought " + "process before the JSON if needed, then return ONLY valid JSON " + "with " + "exactly these keys, in this exact order: {{\"username\": " + "\"\", \"bio\": \"\", \"activity_weight\": }}.\nDo not include markdown, comments, extra keys, or " + "literal placeholder values.\n\n{}", + *validation_error, retry_context); + } + + if (logger_) { + logger_->Log({.level = LogLevel::Error, .phase = PipelinePhase::UserGeneration, .message = std::format( - "LlamaGenerator: malformed user JSON (attempt {}): {}", - attempt + 1, *validation_error)}); - } - - user_prompt = std::format( - "Your previous response was invalid. Error: {}\nReturn the thought " - "process before the JSON if needed, then return ONLY valid JSON " - "with " - "exactly these keys, in this exact order: {{\"username\": " - "\"\", \"bio\": \"\", \"activity_weight\": }}.\nDo not include markdown, comments, extra keys, or " - "literal placeholder values.\n\n{}", - *validation_error, retry_context); - } - - if (logger_) { - logger_->Log({.level = LogLevel::Error, - .phase = PipelinePhase::UserGeneration, - .message = std::format( - "LlamaGenerator: malformed user response " - "after {} attempts: {}", - max_attempts, last_error.empty() ? raw : last_error)}); - } - throw std::runtime_error("LlamaGenerator: malformed user response"); + "LlamaGenerator: malformed user response " + "after {} attempts: {}", + max_attempts, last_error.empty() ? raw : last_error)}); + } + throw std::runtime_error("LlamaGenerator: malformed user response"); } diff --git a/tooling/pipeline/src/data_generation/llama/helpers.cc b/tooling/pipeline/src/data_generation/llama/helpers.cc index 1d71d88e..09ea5505 100644 --- a/tooling/pipeline/src/data_generation/llama/helpers.cc +++ b/tooling/pipeline/src/data_generation/llama/helpers.cc @@ -4,6 +4,8 @@ * parsing, token decoding, and JSON validation helpers for Llama modules. */ +#include + #include #include #include @@ -15,8 +17,6 @@ #include #include -#include - #include "data_generation/llama_generator_helpers.h" namespace { @@ -24,14 +24,14 @@ namespace { * String trimming: removes leading and trailing whitespace */ std::string Trim(std::string_view value) { - constexpr std::string_view whitespace = " \t\n\r\f\v"; - const size_t first_index = value.find_first_not_of(whitespace); - if (first_index == std::string_view::npos) { - return {}; - } + constexpr std::string_view whitespace = " \t\n\r\f\v"; + const size_t first_index = value.find_first_not_of(whitespace); + if (first_index == std::string_view::npos) { + return {}; + } - const size_t last_index = value.find_last_not_of(whitespace); - return std::string(value.substr(first_index, last_index - first_index + 1)); + const size_t last_index = value.find_last_not_of(whitespace); + return std::string(value.substr(first_index, last_index - first_index + 1)); } /** @@ -39,26 +39,26 @@ std::string Trim(std::string_view value) { * spaces */ std::string CondenseWhitespace(std::string_view text) { - std::string out; - out.reserve(text.size()); + std::string out; + out.reserve(text.size()); - bool pending_space = false; - for (const char chr : text) { - if (std::isspace(static_cast(chr)) != 0) { - if (!out.empty()) { - pending_space = true; + bool pending_space = false; + for (const char chr : text) { + if (std::isspace(static_cast(chr)) != 0) { + if (!out.empty()) { + pending_space = true; + } + continue; } - continue; - } - if (pending_space) { - out.push_back(' '); - pending_space = false; - } - out.push_back(chr); - } + if (pending_space) { + out.push_back(' '); + pending_space = false; + } + out.push_back(chr); + } - return out; + return out; } // Guard against truncating in the first half of the string. @@ -69,30 +69,30 @@ constexpr size_t kTruncationGuardDivisor = 2; bool ReadRequiredTrimmedStringField(const boost::json::object& obj, std::string_view key, std::string& out, std::string* error_out) { - const boost::json::value* field = obj.if_contains(key); - if (field == nullptr || !field->is_string()) { - return false; - } + const boost::json::value* field = obj.if_contains(key); + if (field == nullptr || !field->is_string()) { + return false; + } - const auto& string_value = field->as_string(); - out = Trim(std::string_view(string_value.data(), string_value.size())); - return !out.empty(); + const auto& string_value = field->as_string(); + out = Trim(std::string_view(string_value.data(), string_value.size())); + return !out.empty(); } bool HasSchemaPlaceholder(std::span values) { - for (const std::string* value : values) { - std::string lowered = *value; - std::ranges::transform(lowered, lowered.begin(), - [](const unsigned char character) { - return static_cast(std::tolower(character)); - }); + for (const std::string* value : values) { + std::string lowered = *value; + std::ranges::transform( + lowered, lowered.begin(), [](const unsigned char character) { + return static_cast(std::tolower(character)); + }); - if (lowered == "string") { - return true; - } - } + if (lowered == "string") { + return true; + } + } - return false; + return false; } } // namespace @@ -102,166 +102,167 @@ bool HasSchemaPlaceholder(std::span values) { */ std::string PrepareRegionContext(std::string_view region_context, const size_t max_chars) { - std::string normalized = CondenseWhitespace(region_context); - if (normalized.size() <= max_chars) { - return normalized; - } + std::string normalized = CondenseWhitespace(region_context); + if (normalized.size() <= max_chars) { + return normalized; + } - normalized.resize(max_chars); - const size_t last_space = normalized.find_last_of(' '); - if (last_space != std::string::npos && - last_space > max_chars / kTruncationGuardDivisor) { - normalized.resize(last_space); - } + normalized.resize(max_chars); + const size_t last_space = normalized.find_last_of(' '); + if (last_space != std::string::npos && + last_space > max_chars / kTruncationGuardDivisor) { + normalized.resize(last_space); + } - normalized += "..."; - return normalized; + normalized += "..."; + return normalized; } void AppendTokenPiece(const llama_vocab* vocab, llama_token token, std::string& output) { - constexpr size_t initial_buffer_size = 256; + constexpr size_t initial_buffer_size = 256; - std::array buffer{}; + std::array buffer{}; - // serialize the sampled token into UTF-8 bytes + // serialize the sampled token into UTF-8 bytes - auto buffer_too_small = [](int32_t result) -> bool { return result < 0; }; + auto buffer_too_small = [](int32_t result) -> bool { return result < 0; }; - int32_t bytes = - llama_token_to_piece(vocab, token, buffer.data(), buffer.size(), 0, true); + int32_t bytes = llama_token_to_piece(vocab, token, buffer.data(), + buffer.size(), 0, true); - if (!buffer_too_small(bytes)) { - // Append the decoded bytes from the stack buffer. - output.append(buffer.data(), static_cast(bytes)); - return; - } + if (!buffer_too_small(bytes)) { + // Append the decoded bytes from the stack buffer. + output.append(buffer.data(), static_cast(bytes)); + return; + } - const int32_t required_size = -bytes; - std::vector dynamic_buffer(static_cast(required_size)); + const int32_t required_size = -bytes; + std::vector dynamic_buffer(static_cast(required_size)); - // Retry token decoding against the larger heap buffer. - bytes = llama_token_to_piece(vocab, token, dynamic_buffer.data(), - static_cast(dynamic_buffer.size()), 0, - true); + // Retry token decoding against the larger heap buffer. + bytes = llama_token_to_piece(vocab, token, dynamic_buffer.data(), + static_cast(dynamic_buffer.size()), 0, + true); - if (!buffer_too_small(bytes)) { - output.append(dynamic_buffer.data(), static_cast(bytes)); - return; - } + if (!buffer_too_small(bytes)) { + output.append(dynamic_buffer.data(), static_cast(bytes)); + return; + } - throw std::runtime_error( - "LlamaGenerator: failed to decode sampled token piece"); + throw std::runtime_error( + "LlamaGenerator: failed to decode sampled token piece"); } std::optional ValidateBreweryJson(const std::string& raw, BreweryResult& brewery_out) { - boost::system::error_code error_code; - const std::string_view raw_view(raw); - const size_t opening_brace = raw_view.find('{'); - if (opening_brace == std::string_view::npos) { - return "JSON parse error: missing opening brace '{'"; - } + boost::system::error_code error_code; + const std::string_view raw_view(raw); + const size_t opening_brace = raw_view.find('{'); + if (opening_brace == std::string_view::npos) { + return "JSON parse error: missing opening brace '{'"; + } - const std::string_view json_payload = raw_view.substr(opening_brace); - boost::json::value json_value = boost::json::parse(json_payload, error_code); - if (error_code) { - return "JSON parse error: " + error_code.message(); - } + const std::string_view json_payload = raw_view.substr(opening_brace); + boost::json::value json_value = boost::json::parse(json_payload, error_code); + if (error_code) { + return "JSON parse error: " + error_code.message(); + } - if (!json_value.is_object()) { - return "JSON root must be an object"; - } + if (!json_value.is_object()) { + return "JSON root must be an object"; + } - const auto& obj = json_value.get_object(); - if (obj.size() != 4) { - return "JSON object must contain exactly four keys"; - } + const auto& obj = json_value.get_object(); + if (obj.size() != 4) { + return "JSON object must contain exactly four keys"; + } - std::string validation_error; - if (!ReadRequiredTrimmedStringField(obj, "name_en", brewery_out.name_en, - &validation_error)) { - return validation_error; - } + std::string validation_error; + if (!ReadRequiredTrimmedStringField(obj, "name_en", brewery_out.name_en, + &validation_error)) { + return validation_error; + } - if (!ReadRequiredTrimmedStringField(obj, "description_en", - brewery_out.description_en, - &validation_error)) { - return validation_error; - } + if (!ReadRequiredTrimmedStringField(obj, "description_en", + brewery_out.description_en, + &validation_error)) { + return validation_error; + } - if (!ReadRequiredTrimmedStringField(obj, "name_local", brewery_out.name_local, - &validation_error)) { - return validation_error; - } + if (!ReadRequiredTrimmedStringField( + obj, "name_local", brewery_out.name_local, &validation_error)) { + return validation_error; + } - if (!ReadRequiredTrimmedStringField(obj, "description_local", - brewery_out.description_local, - &validation_error)) { - return validation_error; - } + if (!ReadRequiredTrimmedStringField(obj, "description_local", + brewery_out.description_local, + &validation_error)) { + return validation_error; + } - const std::array schema_placeholders = { - &brewery_out.name_en, &brewery_out.description_en, - &brewery_out.name_local, &brewery_out.description_local}; - if (HasSchemaPlaceholder(schema_placeholders)) { - return "JSON appears to be a schema placeholder, not content"; - } + const std::array schema_placeholders = { + &brewery_out.name_en, &brewery_out.description_en, + &brewery_out.name_local, &brewery_out.description_local}; + if (HasSchemaPlaceholder(schema_placeholders)) { + return "JSON appears to be a schema placeholder, not content"; + } - return std::nullopt; + return std::nullopt; } std::optional ValidateUserJson(const std::string& raw, UserResult& user_out) { - boost::system::error_code error_code; - const std::string_view raw_view(raw); - const size_t opening_brace = raw_view.find('{'); - if (opening_brace == std::string_view::npos) { - return "JSON parse error: missing opening brace '{'"; - } + boost::system::error_code error_code; + const std::string_view raw_view(raw); + const size_t opening_brace = raw_view.find('{'); + if (opening_brace == std::string_view::npos) { + return "JSON parse error: missing opening brace '{'"; + } - const std::string_view json_payload = raw_view.substr(opening_brace); - boost::json::value json_value = boost::json::parse(json_payload, error_code); - if (error_code) { - return "JSON parse error: " + error_code.message(); - } + const std::string_view json_payload = raw_view.substr(opening_brace); + boost::json::value json_value = boost::json::parse(json_payload, error_code); + if (error_code) { + return "JSON parse error: " + error_code.message(); + } - if (!json_value.is_object()) { - return "JSON root must be an object"; - } + if (!json_value.is_object()) { + return "JSON root must be an object"; + } - const auto& obj = json_value.get_object(); - if (obj.size() != 3) { - return "JSON object must contain exactly three keys"; - } + const auto& obj = json_value.get_object(); + if (obj.size() != 3) { + return "JSON object must contain exactly three keys"; + } - std::string validation_error; - if (!ReadRequiredTrimmedStringField(obj, "username", user_out.username, - &validation_error)) { - return validation_error; - } + std::string validation_error; + if (!ReadRequiredTrimmedStringField(obj, "username", user_out.username, + &validation_error)) { + return validation_error; + } - if (!ReadRequiredTrimmedStringField(obj, "bio", user_out.bio, - &validation_error)) { - return validation_error; - } + if (!ReadRequiredTrimmedStringField(obj, "bio", user_out.bio, + &validation_error)) { + return validation_error; + } - const boost::json::value* activity_weight_field = - obj.if_contains("activity_weight"); - if (activity_weight_field == nullptr || !activity_weight_field->is_number()) { - return "Missing or invalid numeric field: activity_weight"; - } + const boost::json::value* activity_weight_field = + obj.if_contains("activity_weight"); + if (activity_weight_field == nullptr || + !activity_weight_field->is_number()) { + return "Missing or invalid numeric field: activity_weight"; + } - const double activity_weight = activity_weight_field->to_number(); - if (activity_weight < 0.0 || activity_weight > 1.0) { - return "activity_weight must be between 0 and 1"; - } - user_out.activity_weight = static_cast(activity_weight); + const double activity_weight = activity_weight_field->to_number(); + if (activity_weight < 0.0 || activity_weight > 1.0) { + return "activity_weight must be between 0 and 1"; + } + user_out.activity_weight = static_cast(activity_weight); - const std::array schema_placeholders = {&user_out.username, &user_out.bio}; - if (HasSchemaPlaceholder(schema_placeholders)) { - return "JSON appears to be a schema placeholder, not content"; - } + const std::array schema_placeholders = {&user_out.username, &user_out.bio}; + if (HasSchemaPlaceholder(schema_placeholders)) { + return "JSON appears to be a schema placeholder, not content"; + } - return std::nullopt; + return std::nullopt; } diff --git a/tooling/pipeline/src/data_generation/llama/infer.cc b/tooling/pipeline/src/data_generation/llama/infer.cc index 53f417fc..2ebecbcb 100644 --- a/tooling/pipeline/src/data_generation/llama/infer.cc +++ b/tooling/pipeline/src/data_generation/llama/infer.cc @@ -5,6 +5,8 @@ * output tokens back to text for system+user chat prompts. */ +#include + #include #include #include @@ -14,8 +16,6 @@ #include #include -#include - #include "data_generation/llama_generator.h" #include "data_generation/llama_generator_helpers.h" @@ -30,48 +30,49 @@ using SamplerHandle = std::unique_ptr; struct SamplerConfig { - float temperature; - uint32_t top_k; - float top_p; - uint32_t seed; + float temperature; + uint32_t top_k; + float top_p; + uint32_t seed; }; SamplerHandle MakeSamplerChain(const llama_vocab* vocab, const SamplerConfig& config, std::string_view grammar) { - const llama_sampler_chain_params sampler_params = - llama_sampler_chain_default_params(); + const llama_sampler_chain_params sampler_params = + llama_sampler_chain_default_params(); - SamplerHandle chain(llama_sampler_chain_init(sampler_params), - llama_sampler_free); - if (!chain) { - throw std::runtime_error("LlamaGenerator: failed to initialize sampler"); - } + SamplerHandle chain(llama_sampler_chain_init(sampler_params), + llama_sampler_free); + if (!chain) { + throw std::runtime_error("LlamaGenerator: failed to initialize sampler"); + } - auto add_sampler = [&](llama_sampler* sampler, const char* error_message) { - if (sampler == nullptr) { - throw std::runtime_error(error_message); - } + auto add_sampler = [&](llama_sampler* sampler, const char* error_message) { + if (sampler == nullptr) { + throw std::runtime_error(error_message); + } - llama_sampler_chain_add(chain.get(), sampler); - }; + llama_sampler_chain_add(chain.get(), sampler); + }; - if (!grammar.empty()) { - const std::string grammar_text(grammar); - add_sampler(llama_sampler_init_grammar(vocab, grammar_text.c_str(), "root"), - "LlamaGenerator: failed to initialize grammar sampler"); - } + if (!grammar.empty()) { + const std::string grammar_text(grammar); + add_sampler( + llama_sampler_init_grammar(vocab, grammar_text.c_str(), "root"), + "LlamaGenerator: failed to initialize grammar sampler"); + } - add_sampler(llama_sampler_init_temp(config.temperature), - "LlamaGenerator: failed to initialize temperature sampler"); - add_sampler(llama_sampler_init_top_k(static_cast(config.top_k)), - "LlamaGenerator: failed to initialize top-k sampler"); - add_sampler(llama_sampler_init_top_p(config.top_p, kTopPMinKeep), - "LlamaGenerator: failed to initialize top-p sampler"); - add_sampler(llama_sampler_init_dist(config.seed), - "LlamaGenerator: failed to initialize distribution sampler"); + add_sampler(llama_sampler_init_temp(config.temperature), + "LlamaGenerator: failed to initialize temperature sampler"); + add_sampler(llama_sampler_init_top_k(static_cast(config.top_k)), + "LlamaGenerator: failed to initialize top-k sampler"); + add_sampler(llama_sampler_init_top_p(config.top_p, kTopPMinKeep), + "LlamaGenerator: failed to initialize top-p sampler"); + add_sampler(llama_sampler_init_dist(config.seed), + "LlamaGenerator: failed to initialize distribution sampler"); - return chain; + return chain; } } // namespace @@ -80,163 +81,164 @@ std::string LlamaGenerator::Infer(const std::string& system_prompt, const std::string& prompt, const int max_tokens, std::string_view grammar) { - return InferFormatted(prompt_formatter_->Format(system_prompt, prompt), - max_tokens, grammar); + return InferFormatted(prompt_formatter_->Format(system_prompt, prompt), + max_tokens, grammar); } std::string LlamaGenerator::InferFormatted(const std::string& formatted_prompt, const int max_tokens, std::string_view grammar) { - /** - * Validate that model and context are loaded - */ - if (!model_ || !context_) { - throw std::runtime_error("LlamaGenerator: model not loaded"); - } + /** + * Validate that model and context are loaded + */ + if (!model_ || !context_) { + throw std::runtime_error("LlamaGenerator: model not loaded"); + } - /** - * Get vocabulary for tokenization and token-to-text conversion - */ - const llama_vocab* vocab = llama_model_get_vocab(model_.get()); - if (vocab == nullptr) { - throw std::runtime_error("LlamaGenerator: vocab unavailable"); - } + /** + * Get vocabulary for tokenization and token-to-text conversion + */ + const llama_vocab* vocab = llama_model_get_vocab(model_.get()); + if (vocab == nullptr) { + throw std::runtime_error("LlamaGenerator: vocab unavailable"); + } - const SamplerConfig sampler_config{ - .temperature = sampling_temperature_, - .top_k = sampling_top_k_, - .top_p = sampling_top_p_, - .seed = static_cast(rng_()), - }; - const auto sampler = MakeSamplerChain(vocab, sampler_config, grammar); + const SamplerConfig sampler_config{ + .temperature = sampling_temperature_, + .top_k = sampling_top_k_, + .top_p = sampling_top_p_, + .seed = static_cast(rng_()), + }; + const auto sampler = MakeSamplerChain(vocab, sampler_config, grammar); - /** - * Clear KV cache to ensure clean inference state (no residual context) - */ - llama_memory_clear(llama_get_memory(context_.get()), true); + /** + * Clear KV cache to ensure clean inference state (no residual context) + */ + llama_memory_clear(llama_get_memory(context_.get()), true); - /** - * TOKENIZATION PHASE - * Convert text prompt into token IDs (integers) that the model understands - */ - std::vector prompt_tokens(formatted_prompt.size() + - kPromptTokenSlack); + /** + * TOKENIZATION PHASE + * Convert text prompt into token IDs (integers) that the model understands + */ + std::vector prompt_tokens(formatted_prompt.size() + + kPromptTokenSlack); - int32_t token_count = llama_tokenize( - vocab, formatted_prompt.c_str(), - static_cast(formatted_prompt.size()), prompt_tokens.data(), - static_cast(prompt_tokens.size()), true, true); + int32_t token_count = llama_tokenize( + vocab, formatted_prompt.c_str(), + static_cast(formatted_prompt.size()), prompt_tokens.data(), + static_cast(prompt_tokens.size()), true, true); - /** - * If buffer too small, negative return indicates required size - */ - if (token_count < 0) { - prompt_tokens.resize(static_cast(-token_count)); + /** + * If buffer too small, negative return indicates required size + */ + if (token_count < 0) { + prompt_tokens.resize(static_cast(-token_count)); - token_count = llama_tokenize( - vocab, formatted_prompt.c_str(), - static_cast(formatted_prompt.size()), prompt_tokens.data(), - static_cast(prompt_tokens.size()), true, true); - } + token_count = llama_tokenize( + vocab, formatted_prompt.c_str(), + static_cast(formatted_prompt.size()), prompt_tokens.data(), + static_cast(prompt_tokens.size()), true, true); + } - if (token_count < 0) { - throw std::runtime_error("LlamaGenerator: prompt tokenization failed"); - } + if (token_count < 0) { + throw std::runtime_error("LlamaGenerator: prompt tokenization failed"); + } - /** - * CONTEXT SIZE VALIDATION - * Validate and compute effective token budgets based on context window - * constraints - */ - const auto n_ctx = static_cast(llama_n_ctx(context_.get())); - const auto n_batch = static_cast(llama_n_batch(context_.get())); - if (n_ctx <= 1 || n_batch <= 0) { - throw std::runtime_error("LlamaGenerator: invalid context or batch size"); - } + /** + * CONTEXT SIZE VALIDATION + * Validate and compute effective token budgets based on context window + * constraints + */ + const auto n_ctx = static_cast(llama_n_ctx(context_.get())); + const auto n_batch = static_cast(llama_n_batch(context_.get())); + if (n_ctx <= 1 || n_batch <= 0) { + throw std::runtime_error("LlamaGenerator: invalid context or batch size"); + } - /** - * Clamp generation limit to available context window, reserve space for - * output - */ - const int32_t effective_max_tokens = - std::max(1, std::min(max_tokens, n_ctx - 1)); + /** + * Clamp generation limit to available context window, reserve space for + * output + */ + const int32_t effective_max_tokens = + std::max(1, std::min(max_tokens, n_ctx - 1)); - /** - * Prompt can use remaining context after reserving space for generation - */ - int32_t prompt_budget = std::min(n_batch, n_ctx - effective_max_tokens); - prompt_budget = std::max(1, prompt_budget); + /** + * Prompt can use remaining context after reserving space for generation + */ + int32_t prompt_budget = std::min(n_batch, n_ctx - effective_max_tokens); + prompt_budget = std::max(1, prompt_budget); - /** - * Truncate prompt if necessary to fit within constraints - */ - prompt_tokens.resize(static_cast(token_count)); - if (token_count > prompt_budget) { - if (logger_) { - logger_->Log({.level = LogLevel::Warn, - .phase = PipelinePhase::BreweryAndBeerGeneration, - .message = std::format( - "LlamaGenerator: prompt too long ({} tokens), " - "truncating to {} tokens to fit n_batch/n_ctx limits", - token_count, prompt_budget)}); - } - prompt_tokens.resize(static_cast(prompt_budget)); - token_count = prompt_budget; - } + /** + * Truncate prompt if necessary to fit within constraints + */ + prompt_tokens.resize(static_cast(token_count)); + if (token_count > prompt_budget) { + if (logger_) { + logger_->Log( + {.level = LogLevel::Warn, + .phase = PipelinePhase::BreweryAndBeerGeneration, + .message = std::format( + "LlamaGenerator: prompt too long ({} tokens), " + "truncating to {} tokens to fit n_batch/n_ctx limits", + token_count, prompt_budget)}); + } + prompt_tokens.resize(static_cast(prompt_budget)); + token_count = prompt_budget; + } - /** - * PROMPT PROCESSING PHASE - * Create a batch containing all prompt tokens and feed through the model - * This computes internal representations and fills the KV cache - */ - const llama_batch prompt_batch = llama_batch_get_one( - prompt_tokens.data(), static_cast(prompt_tokens.size())); - if (llama_decode(context_.get(), prompt_batch) != 0) { - throw std::runtime_error("LlamaGenerator: prompt decode failed"); - } + /** + * PROMPT PROCESSING PHASE + * Create a batch containing all prompt tokens and feed through the model + * This computes internal representations and fills the KV cache + */ + const llama_batch prompt_batch = llama_batch_get_one( + prompt_tokens.data(), static_cast(prompt_tokens.size())); + if (llama_decode(context_.get(), prompt_batch) != 0) { + throw std::runtime_error("LlamaGenerator: prompt decode failed"); + } - /** - * TOKEN GENERATION LOOP - * Iteratively generate tokens one at a time until max_tokens or - * end-of-sequence - */ - std::vector generated_tokens; - generated_tokens.reserve(static_cast(effective_max_tokens)); + /** + * TOKEN GENERATION LOOP + * Iteratively generate tokens one at a time until max_tokens or + * end-of-sequence + */ + std::vector generated_tokens; + generated_tokens.reserve(static_cast(effective_max_tokens)); - for (int i = 0; i < effective_max_tokens; ++i) { - /** - * Sample next token using configured sampler chain and model logits - * Index -1 means use the last output position from previous batch - */ - const llama_token next = - llama_sampler_sample(sampler.get(), context_.get(), -1); - /** - * Stop if model predicts end-of-generation token (EOS/EOT) - */ - if (llama_vocab_is_eog(vocab, next)) { - break; - } - generated_tokens.push_back(next); - /** - * Feed the sampled token back into model for next iteration - * (autoregressive) - */ - llama_token decode_token = next; - const llama_batch one_token_batch = llama_batch_get_one(&decode_token, 1); - if (llama_decode(context_.get(), one_token_batch) != 0) { - throw std::runtime_error( - "LlamaGenerator: decode failed during generation"); - } - } + for (int i = 0; i < effective_max_tokens; ++i) { + /** + * Sample next token using configured sampler chain and model logits + * Index -1 means use the last output position from previous batch + */ + const llama_token next = + llama_sampler_sample(sampler.get(), context_.get(), -1); + /** + * Stop if model predicts end-of-generation token (EOS/EOT) + */ + if (llama_vocab_is_eog(vocab, next)) { + break; + } + generated_tokens.push_back(next); + /** + * Feed the sampled token back into model for next iteration + * (autoregressive) + */ + llama_token decode_token = next; + const llama_batch one_token_batch = llama_batch_get_one(&decode_token, 1); + if (llama_decode(context_.get(), one_token_batch) != 0) { + throw std::runtime_error( + "LlamaGenerator: decode failed during generation"); + } + } - /** - * DETOKENIZATION PHASE - * Convert generated token IDs back to text using vocabulary - */ - std::string output; - for (const llama_token token : generated_tokens) { - AppendTokenPiece(vocab, token, output); - } + /** + * DETOKENIZATION PHASE + * Convert generated token IDs back to text using vocabulary + */ + std::string output; + for (const llama_token token : generated_tokens) { + AppendTokenPiece(vocab, token, output); + } - return output; + return output; } diff --git a/tooling/pipeline/src/data_generation/llama/llama_generator.cc b/tooling/pipeline/src/data_generation/llama/llama_generator.cc index 3dd61ed6..c18c9789 100644 --- a/tooling/pipeline/src/data_generation/llama/llama_generator.cc +++ b/tooling/pipeline/src/data_generation/llama/llama_generator.cc @@ -5,30 +5,30 @@ #include "data_generation/llama_generator.h" +#include + #include #include #include #include #include -#include - #include "data_model/models.h" static constexpr uint32_t kMaxContextSize = 32768U; void LlamaGenerator::ModelDeleter::operator()( llama_model* model) const noexcept { - if (model != nullptr) { - llama_model_free(model); - } + if (model != nullptr) { + llama_model_free(model); + } } void LlamaGenerator::ContextDeleter::operator()( llama_context* context) const noexcept { - if (context != nullptr) { - llama_free(context); - } + if (context != nullptr) { + llama_free(context); + } } LlamaGenerator::LlamaGenerator( @@ -40,61 +40,61 @@ LlamaGenerator::LlamaGenerator( logger_(std::move(logger)), prompt_formatter_(std::move(prompt_formatter)), prompt_directory_(std::move(prompt_directory)) { - if (model_path.empty()) { - throw std::runtime_error("LlamaGenerator: model path must not be empty"); - } + if (model_path.empty()) { + throw std::runtime_error("LlamaGenerator: model path must not be empty"); + } - if (!prompt_formatter_) { - throw std::runtime_error( - "LlamaGenerator: prompt formatter dependency must not be null"); - } + if (!prompt_formatter_) { + throw std::runtime_error( + "LlamaGenerator: prompt formatter dependency must not be null"); + } - if (!prompt_directory_) { - throw std::runtime_error( - "LlamaGenerator: prompt directory dependency must not be null"); - } + if (!prompt_directory_) { + throw std::runtime_error( + "LlamaGenerator: prompt directory dependency must not be null"); + } - const auto sampling = options.generator.sampling.value_or(SamplingOptions{}); + const auto sampling = options.generator.sampling.value_or(SamplingOptions{}); - if (sampling.temperature < 0.0F) { - throw std::runtime_error( - "LlamaGenerator: sampling temperature must be >= 0"); - } + if (sampling.temperature < 0.0F) { + throw std::runtime_error( + "LlamaGenerator: sampling temperature must be >= 0"); + } - if (sampling.top_p <= 0.0F || sampling.top_p > 1.0F) { - throw std::runtime_error( - "LlamaGenerator: sampling top-p must be in (0, 1]"); - } + if (sampling.top_p <= 0.0F || sampling.top_p > 1.0F) { + throw std::runtime_error( + "LlamaGenerator: sampling top-p must be in (0, 1]"); + } - if (sampling.top_k == 0U) { - throw std::runtime_error("LlamaGenerator: sampling top-k must be > 0"); - } + if (sampling.top_k == 0U) { + throw std::runtime_error("LlamaGenerator: sampling top-k must be > 0"); + } - if (sampling.seed < -1) { - throw std::runtime_error( - "LlamaGenerator: seed must be >= 0, or -1 for random"); - } + if (sampling.seed < -1) { + throw std::runtime_error( + "LlamaGenerator: seed must be >= 0, or -1 for random"); + } - if (sampling.n_ctx == 0 || sampling.n_ctx > kMaxContextSize) { - throw std::runtime_error( - "LlamaGenerator: context size must be in range [1, 32768]"); - } + if (sampling.n_ctx == 0 || sampling.n_ctx > kMaxContextSize) { + throw std::runtime_error( + "LlamaGenerator: context size must be in range [1, 32768]"); + } - sampling_temperature_ = sampling.temperature; - sampling_top_p_ = sampling.top_p; - sampling_top_k_ = sampling.top_k; + sampling_temperature_ = sampling.temperature; + sampling_top_p_ = sampling.top_p; + sampling_top_k_ = sampling.top_k; - if (sampling.seed == -1) { - std::random_device random_device; - rng_.seed(random_device()); - } else { - rng_.seed(static_cast(sampling.seed)); - } + if (sampling.seed == -1) { + std::random_device random_device; + rng_.seed(random_device()); + } else { + rng_.seed(static_cast(sampling.seed)); + } - n_ctx_ = sampling.n_ctx; - n_gpu_layers_ = sampling.n_gpu_layers; + n_ctx_ = sampling.n_ctx; + n_gpu_layers_ = sampling.n_gpu_layers; - this->Load(model_path); + this->Load(model_path); } LlamaGenerator::~LlamaGenerator() = default; diff --git a/tooling/pipeline/src/data_generation/llama/load.cc b/tooling/pipeline/src/data_generation/llama/load.cc index d5a33089..33192a79 100644 --- a/tooling/pipeline/src/data_generation/llama/load.cc +++ b/tooling/pipeline/src/data_generation/llama/load.cc @@ -4,15 +4,15 @@ * context, and resets prior resources during model initialization. */ +#include +#include + #include #include #include #include #include -#include -#include - #include "data_generation/llama_generator.h" // Maximum batch size for decode operations. Capping the batch prevents @@ -20,42 +20,42 @@ static constexpr uint32_t kMaxBatchSize = 5000U; void LlamaGenerator::Load(const std::string& model_path) { - context_.reset(); - model_.reset(); + context_.reset(); + model_.reset(); - // Specifically load dynamic ggml backends (like CUDA) that are provided - // externally before attempting to load a model. - ggml_backend_load_all(); + // Specifically load dynamic ggml backends (like CUDA) that are provided + // externally before attempting to load a model. + ggml_backend_load_all(); - llama_model_params model_params = llama_model_default_params(); - model_params.n_gpu_layers = n_gpu_layers_; + llama_model_params model_params = llama_model_default_params(); + model_params.n_gpu_layers = n_gpu_layers_; - ModelHandle loaded_model( - llama_model_load_from_file(model_path.c_str(), model_params)); + ModelHandle loaded_model( + llama_model_load_from_file(model_path.c_str(), model_params)); - if (!loaded_model) { - throw std::runtime_error( - "LlamaGenerator: failed to load model from path: " + model_path); - } + if (!loaded_model) { + throw std::runtime_error( + "LlamaGenerator: failed to load model from path: " + model_path); + } - llama_context_params context_params = llama_context_default_params(); - context_params.n_ctx = n_ctx_; - context_params.n_batch = std::min(n_ctx_, kMaxBatchSize); + llama_context_params context_params = llama_context_default_params(); + context_params.n_ctx = n_ctx_; + context_params.n_batch = std::min(n_ctx_, kMaxBatchSize); - ContextHandle loaded_context( - llama_init_from_model(loaded_model.get(), context_params)); + ContextHandle loaded_context( + llama_init_from_model(loaded_model.get(), context_params)); - if (!loaded_context) { - throw std::runtime_error("LlamaGenerator: failed to create context"); - } + if (!loaded_context) { + throw std::runtime_error("LlamaGenerator: failed to create context"); + } - model_ = std::move(loaded_model); - context_ = std::move(loaded_context); + model_ = std::move(loaded_model); + context_ = std::move(loaded_context); - if (logger_) { - logger_->Log({.level = LogLevel::Info, - .phase = PipelinePhase::Startup, - .message = std::format("[LlamaGenerator] Loaded model: {} ", - model_path)}); - } + if (logger_) { + logger_->Log({.level = LogLevel::Info, + .phase = PipelinePhase::Startup, + .message = std::format("[LlamaGenerator] Loaded model: {} ", + model_path)}); + } }