AaronPo/the-biergarten-app
1
0
Fork 0
mirror of https://github.com/aaronpo97/the-biergarten-app.git synced 2026-10-15 01:54:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Formatted src/data_generation/llama

Browse Source
This commit is contained in:
Aaron Po
2026-07-19 04:10:33 -04:00
parent 1885d9859f
commit d723f53cba
6 changed files with 622 additions and 617 deletions
Download Patch File Download Diff File Expand all files Collapse all files

236
tooling/pipeline/src/data_generation/llama/generate_brewery.cc
View File

@@ -18,144 +18,144 @@
static std::string FormatLocalLanguageCodes( static std::string FormatLocalLanguageCodes(
const std::vector<std::string>& codes) { const std::vector<std::string>& codes) {
if (codes.empty()) { if (codes.empty()) {
return "Not provided"; return "Not provided";
} }
std::string formatted; std::string formatted;
for (const std::string& code : codes) { for (const std::string& code : codes) {
if (!formatted.empty()) { if (!formatted.empty()) {
formatted += ", "; formatted += ", ";
} }
formatted += code; formatted += code;
} }
return formatted; return formatted;
} }
static constexpr int kBreweryInitialMaxTokens = 2800; static constexpr int kBreweryInitialMaxTokens = 2800;
BreweryResult LlamaGenerator::GenerateBrewery( BreweryResult LlamaGenerator::GenerateBrewery(
const EnrichedCity& enriched_city) { const EnrichedCity& enriched_city) {
const City& location = enriched_city.location; const City& location = enriched_city.location;
const std::string& region_context = enriched_city.region_context; const std::string& region_context = enriched_city.region_context;
/** /**
* Preprocess and truncate region context to manageable size * Preprocess and truncate region context to manageable size
*/ */
const std::string safe_region_context = PrepareRegionContext(region_context); const std::string safe_region_context = PrepareRegionContext(region_context);
const std::string local_language_codes = const std::string local_language_codes =
FormatLocalLanguageCodes(location.local_languages); FormatLocalLanguageCodes(location.local_languages);
const std::string country_suffix = const std::string country_suffix =
location.country.empty() ? std::string{} location.country.empty() ? std::string{}
: std::format(", {}", location.country); : std::format(", {}", location.country);
/** /**
* Load brewery system prompt via the injected prompt directory. * Load brewery system prompt via the injected prompt directory.
* The key "BREWERY_GENERATION" resolves to BREWERY_GENERATION.md inside * The key "BREWERY_GENERATION" resolves to BREWERY_GENERATION.md inside
* the configured --prompt-dir. Throws on missing or empty file. * the configured --prompt-dir. Throws on missing or empty file.
*/ */
const std::string system_prompt = const std::string system_prompt =
prompt_directory_->Load("BREWERY_GENERATION"); prompt_directory_->Load("BREWERY_GENERATION");
std::string user_prompt = std::format( std::string user_prompt = std::format(
"## CITY:\n{}\n\n## COUNTRY:\n{}\n\n## LOCAL LANGUAGE CODES:\n{}\n\n## " "## CITY:\n{}\n\n## COUNTRY:\n{}\n\n## LOCAL LANGUAGE CODES:\n{}\n\n## "
"CONTEXT:\n{}", "CONTEXT:\n{}",
location.city, location.country, local_language_codes, location.city, location.country, local_language_codes,
safe_region_context); safe_region_context);
/** /**
* Store location context for retry prompts (without repeating full context) * Store location context for retry prompts (without repeating full context)
*/ */
const std::string retry_location = const std::string retry_location =
std::format("City: {}{}\nLocal language codes: {}", location.city, std::format("City: {}{}\nLocal language codes: {}", location.city,
country_suffix, local_language_codes); country_suffix, local_language_codes);
/** /**
* RETRY LOOP with validation and error correction * RETRY LOOP with validation and error correction
* Attempts to generate valid brewery data up to 3 times, with * Attempts to generate valid brewery data up to 3 times, with
* feedback-based refinement * feedback-based refinement
*/ */
constexpr int max_attempts = 3; constexpr int max_attempts = 3;
std::string raw; std::string raw;
std::string last_error; std::string last_error;
// Token budget: too small risks truncating valid JSON mid-string. // Token budget: too small risks truncating valid JSON mid-string.
int max_tokens = kBreweryInitialMaxTokens; int max_tokens = kBreweryInitialMaxTokens;
// Limit output length to keep it concise and focused
for (int attempt = 0; attempt < max_attempts; ++attempt) {
// Generate brewery data from LLM
raw = this->Infer(system_prompt, user_prompt, max_tokens,
kBreweryJsonGrammar);
if (logger_) {
logger_->Log(
{.level = LogLevel::Debug,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format("LlamaGenerator: raw output (attempt {}): {}",
attempt + 1, raw)});
}
// Validate output: parse JSON and check required fields
BreweryResult brewery;
const std::optional<std::string> validation_error =
ValidateBreweryJson(raw, brewery);
if (!validation_error.has_value()) {
// Success: return parsed brewery data
// Limit output length to keep it concise and focused
for (int attempt = 0; attempt < max_attempts; ++attempt) {
// Generate brewery data from LLM
raw = this->Infer(system_prompt, user_prompt, max_tokens,
kBreweryJsonGrammar);
if (logger_) { if (logger_) {
logger_->Log( logger_->Log({.level = LogLevel::Debug,
{.level = LogLevel::Info, .phase = PipelinePhase::BreweryAndBeerGeneration,
.phase = PipelinePhase::BreweryAndBeerGeneration, .message = std::format(
.message = std::format("LlamaGenerator: successfully generated " "LlamaGenerator: raw output (attempt {}): {}",
"brewery data on attempt {}", attempt + 1, raw)});
attempt + 1)});
} }
return brewery; // Validate output: parse JSON and check required fields
}
// Validation failed: log error and prepare corrective feedback BreweryResult brewery;
const std::optional<std::string> validation_error =
ValidateBreweryJson(raw, brewery);
last_error = *validation_error; if (!validation_error.has_value()) {
if (logger_) { // Success: return parsed brewery data
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"LlamaGenerator: malformed brewery JSON (attempt {}): {}",
attempt + 1, *validation_error)});
}
// Update prompt with error details to guide LLM toward correct output. if (logger_) {
user_prompt = std::format( logger_->Log({.level = LogLevel::Info,
"Your previous response was invalid. Error: {}\nReturn the thought " .phase = PipelinePhase::BreweryAndBeerGeneration,
"process before the JSON if needed, then return ONLY valid JSON " .message = std::format(
"with " "LlamaGenerator: successfully generated "
"exactly these keys, in this exact order: {{\"name_en\": " "brewery data on attempt {}",
"\"<English " attempt + 1)});
"brewery name>\", \"description_en\": \"<English single-paragraph " }
"description>\", \"name_local\": \"<local-language brewery "
"name>\", "
"\"description_local\": \"<local-language single-paragraph "
"description>\"}}.\nDo not include markdown, comments, extra keys, "
"or "
"literal placeholder values.\n\nKeep the JSON strings concise "
"enough "
"to fit within the token budget.\n\n{}",
*validation_error, retry_location);
}
// All retry attempts exhausted: log failure and throw exception return brewery;
if (logger_) { }
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::BreweryAndBeerGeneration, // Validation failed: log error and prepare corrective feedback
.message = std::format(
"LlamaGenerator: malformed brewery " last_error = *validation_error;
"response after {} attempts: {}", if (logger_) {
max_attempts, last_error.empty() ? raw : last_error)}); logger_->Log(
} {.level = LogLevel::Warn,
throw std::runtime_error("LlamaGenerator: malformed brewery response"); .phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"LlamaGenerator: malformed brewery JSON (attempt {}): {}",
attempt + 1, *validation_error)});
}
// Update prompt with error details to guide LLM toward correct output.
user_prompt = std::format(
"Your previous response was invalid. Error: {}\nReturn the thought "
"process before the JSON if needed, then return ONLY valid JSON "
"with "
"exactly these keys, in this exact order: {{\"name_en\": "
"\"<English "
"brewery name>\", \"description_en\": \"<English single-paragraph "
"description>\", \"name_local\": \"<local-language brewery "
"name>\", "
"\"description_local\": \"<local-language single-paragraph "
"description>\"}}.\nDo not include markdown, comments, extra keys, "
"or "
"literal placeholder values.\n\nKeep the JSON strings concise "
"enough "
"to fit within the token budget.\n\n{}",
*validation_error, retry_location);
}
// All retry attempts exhausted: log failure and throw exception
if (logger_) {
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"LlamaGenerator: malformed brewery "
"response after {} attempts: {}",
max_attempts, last_error.empty() ? raw : last_error)});
}
throw std::runtime_error("LlamaGenerator: malformed brewery response");
} }

166
tooling/pipeline/src/data_generation/llama/generate_user.cc
View File

@@ -19,97 +19,99 @@ static constexpr int kUserInitialMaxTokens = 1200;
UserResult LlamaGenerator::GenerateUser(const EnrichedCity& city, UserResult LlamaGenerator::GenerateUser(const EnrichedCity& city,
const UserPersona& persona, const UserPersona& persona,
const Name& name) { const Name& name) {
std::string style_affinities; std::string style_affinities;
for (const std::string& style : persona.style_affinities) { for (const std::string& style : persona.style_affinities) {
if (!style_affinities.empty()) { if (!style_affinities.empty()) {
style_affinities += ", "; style_affinities += ", ";
} }
style_affinities += style; style_affinities += style;
} }
const std::string system_prompt = prompt_directory_->Load("USER_GENERATION"); const std::string system_prompt = prompt_directory_->Load("USER_GENERATION");
std::string user_prompt = std::format( std::string user_prompt = std::format(
"## NAME:\n\n{} {}\n\n" "## NAME:\n\n{} {}\n\n"
"## GENDER:\n\n{}\n\n" "## GENDER:\n\n{}\n\n"
"## CITY:\n\n{}\n\n" "## CITY:\n\n{}\n\n"
"## COUNTRY:\n\n{}\n\n" "## COUNTRY:\n\n{}\n\n"
"## PERSONA:\n\n{}\n\n" "## PERSONA:\n\n{}\n\n"
"## PERSONA DESCRIPTION:\n\n{}\n\n" "## PERSONA DESCRIPTION:\n\n{}\n\n"
"## STYLE AFFINITIES:\n\n{}", "## STYLE AFFINITIES:\n\n{}",
name.first_name, name.last_name, name.gender, city.location.city, name.first_name, name.last_name, name.gender, city.location.city,
city.location.country, persona.name, persona.description, city.location.country, persona.name, persona.description,
style_affinities); style_affinities);
const std::string retry_context = std::format( const std::string retry_context = std::format(
"Name: {} {}\nCity: {}, {}\nPersona: {}", name.first_name, name.last_name, "Name: {} {}\nCity: {}, {}\nPersona: {}", name.first_name,
city.location.city, city.location.country, persona.name); name.last_name, city.location.city, city.location.country, persona.name);
constexpr int max_attempts = 3; constexpr int max_attempts = 3;
std::string raw; std::string raw;
std::string last_error; std::string last_error;
int max_tokens = kUserInitialMaxTokens; int max_tokens = kUserInitialMaxTokens;
for (int attempt = 0; attempt < max_attempts; ++attempt) { for (int attempt = 0; attempt < max_attempts; ++attempt) {
raw = this->Infer(system_prompt, user_prompt, max_tokens, kUserJsonGrammar); raw =
if (logger_) { this->Infer(system_prompt, user_prompt, max_tokens, kUserJsonGrammar);
logger_->Log(
{.level = LogLevel::Debug,
.phase = PipelinePhase::UserGeneration,
.message = std::format("LlamaGenerator: raw output (attempt {}): {}",
attempt + 1, raw)});
}
UserResult user;
const std::optional<std::string> validation_error =
ValidateUserJson(raw, user);
if (!validation_error.has_value()) {
if (logger_) { if (logger_) {
logger_->Log( logger_->Log({.level = LogLevel::Debug,
{.level = LogLevel::Info, .phase = PipelinePhase::UserGeneration,
.phase = PipelinePhase::UserGeneration, .message = std::format(
.message = std::format("LlamaGenerator: successfully " "LlamaGenerator: raw output (attempt {}): {}",
"generated user data on attempt {}", attempt + 1, raw)});
attempt + 1)});
} }
user.first_name = name.first_name; UserResult user;
user.last_name = name.last_name; const std::optional<std::string> validation_error =
user.gender = name.gender; ValidateUserJson(raw, user);
return user;
}
last_error = *validation_error; if (!validation_error.has_value()) {
if (logger_) { if (logger_) {
logger_->Log({.level = LogLevel::Warn, logger_->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::UserGeneration,
.message = std::format("LlamaGenerator: successfully "
"generated user data on attempt {}",
attempt + 1)});
}
user.first_name = name.first_name;
user.last_name = name.last_name;
user.gender = name.gender;
return user;
}
last_error = *validation_error;
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"LlamaGenerator: malformed user JSON (attempt {}): {}",
attempt + 1, *validation_error)});
}
user_prompt = std::format(
"Your previous response was invalid. Error: {}\nReturn the thought "
"process before the JSON if needed, then return ONLY valid JSON "
"with "
"exactly these keys, in this exact order: {{\"username\": "
"\"<handle "
"derived from the given name>\", \"bio\": \"<first-person bio "
"grounded in the persona>\", \"activity_weight\": <number between "
"0 "
"and 1>}}.\nDo not include markdown, comments, extra keys, or "
"literal placeholder values.\n\n{}",
*validation_error, retry_context);
}
if (logger_) {
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::UserGeneration, .phase = PipelinePhase::UserGeneration,
.message = std::format( .message = std::format(
"LlamaGenerator: malformed user JSON (attempt {}): {}", "LlamaGenerator: malformed user response "
attempt + 1, *validation_error)}); "after {} attempts: {}",
} max_attempts, last_error.empty() ? raw : last_error)});
}
user_prompt = std::format( throw std::runtime_error("LlamaGenerator: malformed user response");
"Your previous response was invalid. Error: {}\nReturn the thought "
"process before the JSON if needed, then return ONLY valid JSON "
"with "
"exactly these keys, in this exact order: {{\"username\": "
"\"<handle "
"derived from the given name>\", \"bio\": \"<first-person bio "
"grounded in the persona>\", \"activity_weight\": <number between "
"0 "
"and 1>}}.\nDo not include markdown, comments, extra keys, or "
"literal placeholder values.\n\n{}",
*validation_error, retry_context);
}
if (logger_) {
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"LlamaGenerator: malformed user response "
"after {} attempts: {}",
max_attempts, last_error.empty() ? raw : last_error)});
}
throw std::runtime_error("LlamaGenerator: malformed user response");
} }

329
tooling/pipeline/src/data_generation/llama/helpers.cc
View File

@@ -4,6 +4,8 @@
* parsing, token decoding, and JSON validation helpers for Llama modules. * parsing, token decoding, and JSON validation helpers for Llama modules.
*/ */
#include <llama.h>
#include <algorithm> #include <algorithm>
#include <array> #include <array>
#include <boost/json.hpp> #include <boost/json.hpp>
@@ -15,8 +17,6 @@
#include <string_view> #include <string_view>
#include <vector> #include <vector>
#include <llama.h>
#include "data_generation/llama_generator_helpers.h" #include "data_generation/llama_generator_helpers.h"
namespace { namespace {
@@ -24,14 +24,14 @@ namespace {
* String trimming: removes leading and trailing whitespace * String trimming: removes leading and trailing whitespace
*/ */
std::string Trim(std::string_view value) { std::string Trim(std::string_view value) {
constexpr std::string_view whitespace = " \t\n\r\f\v"; constexpr std::string_view whitespace = " \t\n\r\f\v";
const size_t first_index = value.find_first_not_of(whitespace); const size_t first_index = value.find_first_not_of(whitespace);
if (first_index == std::string_view::npos) { if (first_index == std::string_view::npos) {
return {}; return {};
} }
const size_t last_index = value.find_last_not_of(whitespace); const size_t last_index = value.find_last_not_of(whitespace);
return std::string(value.substr(first_index, last_index - first_index + 1)); return std::string(value.substr(first_index, last_index - first_index + 1));
} }
/** /**
@@ -39,26 +39,26 @@ std::string Trim(std::string_view value) {
* spaces * spaces
*/ */
std::string CondenseWhitespace(std::string_view text) { std::string CondenseWhitespace(std::string_view text) {
std::string out; std::string out;
out.reserve(text.size()); out.reserve(text.size());
bool pending_space = false; bool pending_space = false;
for (const char chr : text) { for (const char chr : text) {
if (std::isspace(static_cast<unsigned char>(chr)) != 0) { if (std::isspace(static_cast<unsigned char>(chr)) != 0) {
if (!out.empty()) { if (!out.empty()) {
pending_space = true; pending_space = true;
}
continue;
} }
continue;
}
if (pending_space) { if (pending_space) {
out.push_back(' '); out.push_back(' ');
pending_space = false; pending_space = false;
} }
out.push_back(chr); out.push_back(chr);
} }
return out; return out;
} }
// Guard against truncating in the first half of the string. // Guard against truncating in the first half of the string.
@@ -69,30 +69,30 @@ constexpr size_t kTruncationGuardDivisor = 2;
bool ReadRequiredTrimmedStringField(const boost::json::object& obj, bool ReadRequiredTrimmedStringField(const boost::json::object& obj,
std::string_view key, std::string& out, std::string_view key, std::string& out,
std::string* error_out) { std::string* error_out) {
const boost::json::value* field = obj.if_contains(key); const boost::json::value* field = obj.if_contains(key);
if (field == nullptr || !field->is_string()) { if (field == nullptr || !field->is_string()) {
return false; return false;
} }
const auto& string_value = field->as_string(); const auto& string_value = field->as_string();
out = Trim(std::string_view(string_value.data(), string_value.size())); out = Trim(std::string_view(string_value.data(), string_value.size()));
return !out.empty(); return !out.empty();
} }
bool HasSchemaPlaceholder(std::span<std::string* const> values) { bool HasSchemaPlaceholder(std::span<std::string* const> values) {
for (const std::string* value : values) { for (const std::string* value : values) {
std::string lowered = *value; std::string lowered = *value;
std::ranges::transform(lowered, lowered.begin(), std::ranges::transform(
[](const unsigned char character) { lowered, lowered.begin(), [](const unsigned char character) {
return static_cast<char>(std::tolower(character)); return static_cast<char>(std::tolower(character));
}); });
if (lowered == "string") { if (lowered == "string") {
return true; return true;
} }
} }
return false; return false;
} }
} // namespace } // namespace
@@ -102,166 +102,167 @@ bool HasSchemaPlaceholder(std::span<std::string* const> values) {
*/ */
std::string PrepareRegionContext(std::string_view region_context, std::string PrepareRegionContext(std::string_view region_context,
const size_t max_chars) { const size_t max_chars) {
std::string normalized = CondenseWhitespace(region_context); std::string normalized = CondenseWhitespace(region_context);
if (normalized.size() <= max_chars) { if (normalized.size() <= max_chars) {
return normalized; return normalized;
} }
normalized.resize(max_chars); normalized.resize(max_chars);
const size_t last_space = normalized.find_last_of(' '); const size_t last_space = normalized.find_last_of(' ');
if (last_space != std::string::npos && if (last_space != std::string::npos &&
last_space > max_chars / kTruncationGuardDivisor) { last_space > max_chars / kTruncationGuardDivisor) {
normalized.resize(last_space); normalized.resize(last_space);
} }
normalized += "..."; normalized += "...";
return normalized; return normalized;
} }
void AppendTokenPiece(const llama_vocab* vocab, llama_token token, void AppendTokenPiece(const llama_vocab* vocab, llama_token token,
std::string& output) { std::string& output) {
constexpr size_t initial_buffer_size = 256; constexpr size_t initial_buffer_size = 256;
std::array<char, initial_buffer_size> buffer{}; std::array<char, initial_buffer_size> buffer{};
// serialize the sampled token into UTF-8 bytes // serialize the sampled token into UTF-8 bytes
auto buffer_too_small = [](int32_t result) -> bool { return result < 0; }; auto buffer_too_small = [](int32_t result) -> bool { return result < 0; };
int32_t bytes = int32_t bytes = llama_token_to_piece(vocab, token, buffer.data(),
llama_token_to_piece(vocab, token, buffer.data(), buffer.size(), 0, true); buffer.size(), 0, true);
if (!buffer_too_small(bytes)) { if (!buffer_too_small(bytes)) {
// Append the decoded bytes from the stack buffer. // Append the decoded bytes from the stack buffer.
output.append(buffer.data(), static_cast<size_t>(bytes)); output.append(buffer.data(), static_cast<size_t>(bytes));
return; return;
} }
const int32_t required_size = -bytes; const int32_t required_size = -bytes;
std::vector<char> dynamic_buffer(static_cast<size_t>(required_size)); std::vector<char> dynamic_buffer(static_cast<size_t>(required_size));
// Retry token decoding against the larger heap buffer. // Retry token decoding against the larger heap buffer.
bytes = llama_token_to_piece(vocab, token, dynamic_buffer.data(), bytes = llama_token_to_piece(vocab, token, dynamic_buffer.data(),
static_cast<int32_t>(dynamic_buffer.size()), 0, static_cast<int32_t>(dynamic_buffer.size()), 0,
true); true);
if (!buffer_too_small(bytes)) { if (!buffer_too_small(bytes)) {
output.append(dynamic_buffer.data(), static_cast<size_t>(bytes)); output.append(dynamic_buffer.data(), static_cast<size_t>(bytes));
return; return;
} }
throw std::runtime_error( throw std::runtime_error(
"LlamaGenerator: failed to decode sampled token piece"); "LlamaGenerator: failed to decode sampled token piece");
} }
std::optional<std::string> ValidateBreweryJson(const std::string& raw, std::optional<std::string> ValidateBreweryJson(const std::string& raw,
BreweryResult& brewery_out) { BreweryResult& brewery_out) {
boost::system::error_code error_code; boost::system::error_code error_code;
const std::string_view raw_view(raw); const std::string_view raw_view(raw);
const size_t opening_brace = raw_view.find('{'); const size_t opening_brace = raw_view.find('{');
if (opening_brace == std::string_view::npos) { if (opening_brace == std::string_view::npos) {
return "JSON parse error: missing opening brace '{'"; return "JSON parse error: missing opening brace '{'";
} }
const std::string_view json_payload = raw_view.substr(opening_brace); const std::string_view json_payload = raw_view.substr(opening_brace);
boost::json::value json_value = boost::json::parse(json_payload, error_code); boost::json::value json_value = boost::json::parse(json_payload, error_code);
if (error_code) { if (error_code) {
return "JSON parse error: " + error_code.message(); return "JSON parse error: " + error_code.message();
} }
if (!json_value.is_object()) { if (!json_value.is_object()) {
return "JSON root must be an object"; return "JSON root must be an object";
} }
const auto& obj = json_value.get_object(); const auto& obj = json_value.get_object();
if (obj.size() != 4) { if (obj.size() != 4) {
return "JSON object must contain exactly four keys"; return "JSON object must contain exactly four keys";
} }
std::string validation_error; std::string validation_error;
if (!ReadRequiredTrimmedStringField(obj, "name_en", brewery_out.name_en, if (!ReadRequiredTrimmedStringField(obj, "name_en", brewery_out.name_en,
&validation_error)) { &validation_error)) {
return validation_error; return validation_error;
} }
if (!ReadRequiredTrimmedStringField(obj, "description_en", if (!ReadRequiredTrimmedStringField(obj, "description_en",
brewery_out.description_en, brewery_out.description_en,
&validation_error)) { &validation_error)) {
return validation_error; return validation_error;
} }
if (!ReadRequiredTrimmedStringField(obj, "name_local", brewery_out.name_local, if (!ReadRequiredTrimmedStringField(
&validation_error)) { obj, "name_local", brewery_out.name_local, &validation_error)) {
return validation_error; return validation_error;
} }
if (!ReadRequiredTrimmedStringField(obj, "description_local", if (!ReadRequiredTrimmedStringField(obj, "description_local",
brewery_out.description_local, brewery_out.description_local,
&validation_error)) { &validation_error)) {
return validation_error; return validation_error;
} }
const std::array schema_placeholders = { const std::array schema_placeholders = {
&brewery_out.name_en, &brewery_out.description_en, &brewery_out.name_en, &brewery_out.description_en,
&brewery_out.name_local, &brewery_out.description_local}; &brewery_out.name_local, &brewery_out.description_local};
if (HasSchemaPlaceholder(schema_placeholders)) { if (HasSchemaPlaceholder(schema_placeholders)) {
return "JSON appears to be a schema placeholder, not content"; return "JSON appears to be a schema placeholder, not content";
} }
return std::nullopt; return std::nullopt;
} }
std::optional<std::string> ValidateUserJson(const std::string& raw, std::optional<std::string> ValidateUserJson(const std::string& raw,
UserResult& user_out) { UserResult& user_out) {
boost::system::error_code error_code; boost::system::error_code error_code;
const std::string_view raw_view(raw); const std::string_view raw_view(raw);
const size_t opening_brace = raw_view.find('{'); const size_t opening_brace = raw_view.find('{');
if (opening_brace == std::string_view::npos) { if (opening_brace == std::string_view::npos) {
return "JSON parse error: missing opening brace '{'"; return "JSON parse error: missing opening brace '{'";
} }
const std::string_view json_payload = raw_view.substr(opening_brace); const std::string_view json_payload = raw_view.substr(opening_brace);
boost::json::value json_value = boost::json::parse(json_payload, error_code); boost::json::value json_value = boost::json::parse(json_payload, error_code);
if (error_code) { if (error_code) {
return "JSON parse error: " + error_code.message(); return "JSON parse error: " + error_code.message();
} }
if (!json_value.is_object()) { if (!json_value.is_object()) {
return "JSON root must be an object"; return "JSON root must be an object";
} }
const auto& obj = json_value.get_object(); const auto& obj = json_value.get_object();
if (obj.size() != 3) { if (obj.size() != 3) {
return "JSON object must contain exactly three keys"; return "JSON object must contain exactly three keys";
} }
std::string validation_error; std::string validation_error;
if (!ReadRequiredTrimmedStringField(obj, "username", user_out.username, if (!ReadRequiredTrimmedStringField(obj, "username", user_out.username,
&validation_error)) { &validation_error)) {
return validation_error; return validation_error;
} }
if (!ReadRequiredTrimmedStringField(obj, "bio", user_out.bio, if (!ReadRequiredTrimmedStringField(obj, "bio", user_out.bio,
&validation_error)) { &validation_error)) {
return validation_error; return validation_error;
} }
const boost::json::value* activity_weight_field = const boost::json::value* activity_weight_field =
obj.if_contains("activity_weight"); obj.if_contains("activity_weight");
if (activity_weight_field == nullptr || !activity_weight_field->is_number()) { if (activity_weight_field == nullptr ||
return "Missing or invalid numeric field: activity_weight"; !activity_weight_field->is_number()) {
} return "Missing or invalid numeric field: activity_weight";
}
const double activity_weight = activity_weight_field->to_number<double>(); const double activity_weight = activity_weight_field->to_number<double>();
if (activity_weight < 0.0 || activity_weight > 1.0) { if (activity_weight < 0.0 || activity_weight > 1.0) {
return "activity_weight must be between 0 and 1"; return "activity_weight must be between 0 and 1";
} }
user_out.activity_weight = static_cast<float>(activity_weight); user_out.activity_weight = static_cast<float>(activity_weight);
const std::array schema_placeholders = {&user_out.username, &user_out.bio}; const std::array schema_placeholders = {&user_out.username, &user_out.bio};
if (HasSchemaPlaceholder(schema_placeholders)) { if (HasSchemaPlaceholder(schema_placeholders)) {
return "JSON appears to be a schema placeholder, not content"; return "JSON appears to be a schema placeholder, not content";
} }
return std::nullopt; return std::nullopt;
} }

342
tooling/pipeline/src/data_generation/llama/infer.cc
View File

@@ -5,6 +5,8 @@
* output tokens back to text for system+user chat prompts. * output tokens back to text for system+user chat prompts.
*/ */
#include <llama.h>
#include <algorithm> #include <algorithm>
#include <chrono> #include <chrono>
#include <format> #include <format>
@@ -14,8 +16,6 @@
#include <string_view> #include <string_view>
#include <vector> #include <vector>
#include <llama.h>
#include "data_generation/llama_generator.h" #include "data_generation/llama_generator.h"
#include "data_generation/llama_generator_helpers.h" #include "data_generation/llama_generator_helpers.h"
@@ -30,48 +30,49 @@ using SamplerHandle =
std::unique_ptr<llama_sampler, decltype(&llama_sampler_free)>; std::unique_ptr<llama_sampler, decltype(&llama_sampler_free)>;
struct SamplerConfig { struct SamplerConfig {
float temperature; float temperature;
uint32_t top_k; uint32_t top_k;
float top_p; float top_p;
uint32_t seed; uint32_t seed;
}; };
SamplerHandle MakeSamplerChain(const llama_vocab* vocab, SamplerHandle MakeSamplerChain(const llama_vocab* vocab,
const SamplerConfig& config, const SamplerConfig& config,
std::string_view grammar) { std::string_view grammar) {
const llama_sampler_chain_params sampler_params = const llama_sampler_chain_params sampler_params =
llama_sampler_chain_default_params(); llama_sampler_chain_default_params();
SamplerHandle chain(llama_sampler_chain_init(sampler_params), SamplerHandle chain(llama_sampler_chain_init(sampler_params),
llama_sampler_free); llama_sampler_free);
if (!chain) { if (!chain) {
throw std::runtime_error("LlamaGenerator: failed to initialize sampler"); throw std::runtime_error("LlamaGenerator: failed to initialize sampler");
} }
auto add_sampler = [&](llama_sampler* sampler, const char* error_message) { auto add_sampler = [&](llama_sampler* sampler, const char* error_message) {
if (sampler == nullptr) { if (sampler == nullptr) {
throw std::runtime_error(error_message); throw std::runtime_error(error_message);
} }
llama_sampler_chain_add(chain.get(), sampler); llama_sampler_chain_add(chain.get(), sampler);
}; };
if (!grammar.empty()) { if (!grammar.empty()) {
const std::string grammar_text(grammar); const std::string grammar_text(grammar);
add_sampler(llama_sampler_init_grammar(vocab, grammar_text.c_str(), "root"), add_sampler(
"LlamaGenerator: failed to initialize grammar sampler"); llama_sampler_init_grammar(vocab, grammar_text.c_str(), "root"),
} "LlamaGenerator: failed to initialize grammar sampler");
}
add_sampler(llama_sampler_init_temp(config.temperature), add_sampler(llama_sampler_init_temp(config.temperature),
"LlamaGenerator: failed to initialize temperature sampler"); "LlamaGenerator: failed to initialize temperature sampler");
add_sampler(llama_sampler_init_top_k(static_cast<int32_t>(config.top_k)), add_sampler(llama_sampler_init_top_k(static_cast<int32_t>(config.top_k)),
"LlamaGenerator: failed to initialize top-k sampler"); "LlamaGenerator: failed to initialize top-k sampler");
add_sampler(llama_sampler_init_top_p(config.top_p, kTopPMinKeep), add_sampler(llama_sampler_init_top_p(config.top_p, kTopPMinKeep),
"LlamaGenerator: failed to initialize top-p sampler"); "LlamaGenerator: failed to initialize top-p sampler");
add_sampler(llama_sampler_init_dist(config.seed), add_sampler(llama_sampler_init_dist(config.seed),
"LlamaGenerator: failed to initialize distribution sampler"); "LlamaGenerator: failed to initialize distribution sampler");
return chain; return chain;
} }
} // namespace } // namespace
@@ -80,163 +81,164 @@ std::string LlamaGenerator::Infer(const std::string& system_prompt,
const std::string& prompt, const std::string& prompt,
const int max_tokens, const int max_tokens,
std::string_view grammar) { std::string_view grammar) {
return InferFormatted(prompt_formatter_->Format(system_prompt, prompt), return InferFormatted(prompt_formatter_->Format(system_prompt, prompt),
max_tokens, grammar); max_tokens, grammar);
} }
std::string LlamaGenerator::InferFormatted(const std::string& formatted_prompt, std::string LlamaGenerator::InferFormatted(const std::string& formatted_prompt,
const int max_tokens, const int max_tokens,
std::string_view grammar) { std::string_view grammar) {
/** /**
* Validate that model and context are loaded * Validate that model and context are loaded
*/ */
if (!model_ || !context_) { if (!model_ || !context_) {
throw std::runtime_error("LlamaGenerator: model not loaded"); throw std::runtime_error("LlamaGenerator: model not loaded");
} }
/** /**
* Get vocabulary for tokenization and token-to-text conversion * Get vocabulary for tokenization and token-to-text conversion
*/ */
const llama_vocab* vocab = llama_model_get_vocab(model_.get()); const llama_vocab* vocab = llama_model_get_vocab(model_.get());
if (vocab == nullptr) { if (vocab == nullptr) {
throw std::runtime_error("LlamaGenerator: vocab unavailable"); throw std::runtime_error("LlamaGenerator: vocab unavailable");
} }
const SamplerConfig sampler_config{ const SamplerConfig sampler_config{
.temperature = sampling_temperature_, .temperature = sampling_temperature_,
.top_k = sampling_top_k_, .top_k = sampling_top_k_,
.top_p = sampling_top_p_, .top_p = sampling_top_p_,
.seed = static_cast<uint32_t>(rng_()), .seed = static_cast<uint32_t>(rng_()),
}; };
const auto sampler = MakeSamplerChain(vocab, sampler_config, grammar); const auto sampler = MakeSamplerChain(vocab, sampler_config, grammar);
/** /**
* Clear KV cache to ensure clean inference state (no residual context) * Clear KV cache to ensure clean inference state (no residual context)
*/ */
llama_memory_clear(llama_get_memory(context_.get()), true); llama_memory_clear(llama_get_memory(context_.get()), true);
/** /**
* TOKENIZATION PHASE * TOKENIZATION PHASE
* Convert text prompt into token IDs (integers) that the model understands * Convert text prompt into token IDs (integers) that the model understands
*/ */
std::vector<llama_token> prompt_tokens(formatted_prompt.size() + std::vector<llama_token> prompt_tokens(formatted_prompt.size() +
kPromptTokenSlack); kPromptTokenSlack);
int32_t token_count = llama_tokenize( int32_t token_count = llama_tokenize(
vocab, formatted_prompt.c_str(), vocab, formatted_prompt.c_str(),
static_cast<int32_t>(formatted_prompt.size()), prompt_tokens.data(), static_cast<int32_t>(formatted_prompt.size()), prompt_tokens.data(),
static_cast<int32_t>(prompt_tokens.size()), true, true); static_cast<int32_t>(prompt_tokens.size()), true, true);
/** /**
* If buffer too small, negative return indicates required size * If buffer too small, negative return indicates required size
*/ */
if (token_count < 0) { if (token_count < 0) {
prompt_tokens.resize(static_cast<size_t>(-token_count)); prompt_tokens.resize(static_cast<size_t>(-token_count));
token_count = llama_tokenize( token_count = llama_tokenize(
vocab, formatted_prompt.c_str(), vocab, formatted_prompt.c_str(),
static_cast<int32_t>(formatted_prompt.size()), prompt_tokens.data(), static_cast<int32_t>(formatted_prompt.size()), prompt_tokens.data(),
static_cast<int32_t>(prompt_tokens.size()), true, true); static_cast<int32_t>(prompt_tokens.size()), true, true);
} }
if (token_count < 0) { if (token_count < 0) {
throw std::runtime_error("LlamaGenerator: prompt tokenization failed"); throw std::runtime_error("LlamaGenerator: prompt tokenization failed");
} }
/** /**
* CONTEXT SIZE VALIDATION * CONTEXT SIZE VALIDATION
* Validate and compute effective token budgets based on context window * Validate and compute effective token budgets based on context window
* constraints * constraints
*/ */
const auto n_ctx = static_cast<int32_t>(llama_n_ctx(context_.get())); const auto n_ctx = static_cast<int32_t>(llama_n_ctx(context_.get()));
const auto n_batch = static_cast<int32_t>(llama_n_batch(context_.get())); const auto n_batch = static_cast<int32_t>(llama_n_batch(context_.get()));
if (n_ctx <= 1 || n_batch <= 0) { if (n_ctx <= 1 || n_batch <= 0) {
throw std::runtime_error("LlamaGenerator: invalid context or batch size"); throw std::runtime_error("LlamaGenerator: invalid context or batch size");
} }
/** /**
* Clamp generation limit to available context window, reserve space for * Clamp generation limit to available context window, reserve space for
* output * output
*/ */
const int32_t effective_max_tokens = const int32_t effective_max_tokens =
std::max(1, std::min(max_tokens, n_ctx - 1)); std::max(1, std::min(max_tokens, n_ctx - 1));
/** /**
* Prompt can use remaining context after reserving space for generation * Prompt can use remaining context after reserving space for generation
*/ */
int32_t prompt_budget = std::min(n_batch, n_ctx - effective_max_tokens); int32_t prompt_budget = std::min(n_batch, n_ctx - effective_max_tokens);
prompt_budget = std::max<int32_t>(1, prompt_budget); prompt_budget = std::max<int32_t>(1, prompt_budget);
/** /**
* Truncate prompt if necessary to fit within constraints * Truncate prompt if necessary to fit within constraints
*/ */
prompt_tokens.resize(static_cast<size_t>(token_count)); prompt_tokens.resize(static_cast<size_t>(token_count));
if (token_count > prompt_budget) { if (token_count > prompt_budget) {
if (logger_) { if (logger_) {
logger_->Log({.level = LogLevel::Warn, logger_->Log(
.phase = PipelinePhase::BreweryAndBeerGeneration, {.level = LogLevel::Warn,
.message = std::format( .phase = PipelinePhase::BreweryAndBeerGeneration,
"LlamaGenerator: prompt too long ({} tokens), " .message = std::format(
"truncating to {} tokens to fit n_batch/n_ctx limits", "LlamaGenerator: prompt too long ({} tokens), "
token_count, prompt_budget)}); "truncating to {} tokens to fit n_batch/n_ctx limits",
} token_count, prompt_budget)});
prompt_tokens.resize(static_cast<size_t>(prompt_budget)); }
token_count = prompt_budget; prompt_tokens.resize(static_cast<size_t>(prompt_budget));
} token_count = prompt_budget;
}
/** /**
* PROMPT PROCESSING PHASE * PROMPT PROCESSING PHASE
* Create a batch containing all prompt tokens and feed through the model * Create a batch containing all prompt tokens and feed through the model
* This computes internal representations and fills the KV cache * This computes internal representations and fills the KV cache
*/ */
const llama_batch prompt_batch = llama_batch_get_one( const llama_batch prompt_batch = llama_batch_get_one(
prompt_tokens.data(), static_cast<int32_t>(prompt_tokens.size())); prompt_tokens.data(), static_cast<int32_t>(prompt_tokens.size()));
if (llama_decode(context_.get(), prompt_batch) != 0) { if (llama_decode(context_.get(), prompt_batch) != 0) {
throw std::runtime_error("LlamaGenerator: prompt decode failed"); throw std::runtime_error("LlamaGenerator: prompt decode failed");
} }
/** /**
* TOKEN GENERATION LOOP * TOKEN GENERATION LOOP
* Iteratively generate tokens one at a time until max_tokens or * Iteratively generate tokens one at a time until max_tokens or
* end-of-sequence * end-of-sequence
*/ */
std::vector<llama_token> generated_tokens; std::vector<llama_token> generated_tokens;
generated_tokens.reserve(static_cast<size_t>(effective_max_tokens)); generated_tokens.reserve(static_cast<size_t>(effective_max_tokens));
for (int i = 0; i < effective_max_tokens; ++i) { for (int i = 0; i < effective_max_tokens; ++i) {
/** /**
* Sample next token using configured sampler chain and model logits * Sample next token using configured sampler chain and model logits
* Index -1 means use the last output position from previous batch * Index -1 means use the last output position from previous batch
*/ */
const llama_token next = const llama_token next =
llama_sampler_sample(sampler.get(), context_.get(), -1); llama_sampler_sample(sampler.get(), context_.get(), -1);
/** /**
* Stop if model predicts end-of-generation token (EOS/EOT) * Stop if model predicts end-of-generation token (EOS/EOT)
*/ */
if (llama_vocab_is_eog(vocab, next)) { if (llama_vocab_is_eog(vocab, next)) {
break; break;
} }
generated_tokens.push_back(next); generated_tokens.push_back(next);
/** /**
* Feed the sampled token back into model for next iteration * Feed the sampled token back into model for next iteration
* (autoregressive) * (autoregressive)
*/ */
llama_token decode_token = next; llama_token decode_token = next;
const llama_batch one_token_batch = llama_batch_get_one(&decode_token, 1); const llama_batch one_token_batch = llama_batch_get_one(&decode_token, 1);
if (llama_decode(context_.get(), one_token_batch) != 0) { if (llama_decode(context_.get(), one_token_batch) != 0) {
throw std::runtime_error( throw std::runtime_error(
"LlamaGenerator: decode failed during generation"); "LlamaGenerator: decode failed during generation");
} }
} }
/** /**
* DETOKENIZATION PHASE * DETOKENIZATION PHASE
* Convert generated token IDs back to text using vocabulary * Convert generated token IDs back to text using vocabulary
*/ */
std::string output; std::string output;
for (const llama_token token : generated_tokens) { for (const llama_token token : generated_tokens) {
AppendTokenPiece(vocab, token, output); AppendTokenPiece(vocab, token, output);
} }
return output; return output;
} }

102
tooling/pipeline/src/data_generation/llama/llama_generator.cc
View File

@@ -5,30 +5,30 @@
#include "data_generation/llama_generator.h" #include "data_generation/llama_generator.h"
#include <llama.h>
#include <filesystem> #include <filesystem>
#include <memory> #include <memory>
#include <random> #include <random>
#include <stdexcept> #include <stdexcept>
#include <string> #include <string>
#include <llama.h>
#include "data_model/models.h" #include "data_model/models.h"
static constexpr uint32_t kMaxContextSize = 32768U; static constexpr uint32_t kMaxContextSize = 32768U;
void LlamaGenerator::ModelDeleter::operator()( void LlamaGenerator::ModelDeleter::operator()(
llama_model* model) const noexcept { llama_model* model) const noexcept {
if (model != nullptr) { if (model != nullptr) {
llama_model_free(model); llama_model_free(model);
} }
} }
void LlamaGenerator::ContextDeleter::operator()( void LlamaGenerator::ContextDeleter::operator()(
llama_context* context) const noexcept { llama_context* context) const noexcept {
if (context != nullptr) { if (context != nullptr) {
llama_free(context); llama_free(context);
} }
} }
LlamaGenerator::LlamaGenerator( LlamaGenerator::LlamaGenerator(
@@ -40,61 +40,61 @@ LlamaGenerator::LlamaGenerator(
logger_(std::move(logger)), logger_(std::move(logger)),
prompt_formatter_(std::move(prompt_formatter)), prompt_formatter_(std::move(prompt_formatter)),
prompt_directory_(std::move(prompt_directory)) { prompt_directory_(std::move(prompt_directory)) {
if (model_path.empty()) { if (model_path.empty()) {
throw std::runtime_error("LlamaGenerator: model path must not be empty"); throw std::runtime_error("LlamaGenerator: model path must not be empty");
} }
if (!prompt_formatter_) { if (!prompt_formatter_) {
throw std::runtime_error( throw std::runtime_error(
"LlamaGenerator: prompt formatter dependency must not be null"); "LlamaGenerator: prompt formatter dependency must not be null");
} }
if (!prompt_directory_) { if (!prompt_directory_) {
throw std::runtime_error( throw std::runtime_error(
"LlamaGenerator: prompt directory dependency must not be null"); "LlamaGenerator: prompt directory dependency must not be null");
} }
const auto sampling = options.generator.sampling.value_or(SamplingOptions{}); const auto sampling = options.generator.sampling.value_or(SamplingOptions{});
if (sampling.temperature < 0.0F) { if (sampling.temperature < 0.0F) {
throw std::runtime_error( throw std::runtime_error(
"LlamaGenerator: sampling temperature must be >= 0"); "LlamaGenerator: sampling temperature must be >= 0");
} }
if (sampling.top_p <= 0.0F || sampling.top_p > 1.0F) { if (sampling.top_p <= 0.0F || sampling.top_p > 1.0F) {
throw std::runtime_error( throw std::runtime_error(
"LlamaGenerator: sampling top-p must be in (0, 1]"); "LlamaGenerator: sampling top-p must be in (0, 1]");
} }
if (sampling.top_k == 0U) { if (sampling.top_k == 0U) {
throw std::runtime_error("LlamaGenerator: sampling top-k must be > 0"); throw std::runtime_error("LlamaGenerator: sampling top-k must be > 0");
} }
if (sampling.seed < -1) { if (sampling.seed < -1) {
throw std::runtime_error( throw std::runtime_error(
"LlamaGenerator: seed must be >= 0, or -1 for random"); "LlamaGenerator: seed must be >= 0, or -1 for random");
} }
if (sampling.n_ctx == 0 || sampling.n_ctx > kMaxContextSize) { if (sampling.n_ctx == 0 || sampling.n_ctx > kMaxContextSize) {
throw std::runtime_error( throw std::runtime_error(
"LlamaGenerator: context size must be in range [1, 32768]"); "LlamaGenerator: context size must be in range [1, 32768]");
} }
sampling_temperature_ = sampling.temperature; sampling_temperature_ = sampling.temperature;
sampling_top_p_ = sampling.top_p; sampling_top_p_ = sampling.top_p;
sampling_top_k_ = sampling.top_k; sampling_top_k_ = sampling.top_k;
if (sampling.seed == -1) { if (sampling.seed == -1) {
std::random_device random_device; std::random_device random_device;
rng_.seed(random_device()); rng_.seed(random_device());
} else { } else {
rng_.seed(static_cast<uint32_t>(sampling.seed)); rng_.seed(static_cast<uint32_t>(sampling.seed));
} }
n_ctx_ = sampling.n_ctx; n_ctx_ = sampling.n_ctx;
n_gpu_layers_ = sampling.n_gpu_layers; n_gpu_layers_ = sampling.n_gpu_layers;
this->Load(model_path); this->Load(model_path);
} }
LlamaGenerator::~LlamaGenerator() = default; LlamaGenerator::~LlamaGenerator() = default;

64
tooling/pipeline/src/data_generation/llama/load.cc
View File

@@ -4,15 +4,15 @@
* context, and resets prior resources during model initialization. * context, and resets prior resources during model initialization.
*/ */
#include <ggml-backend.h>
#include <llama.h>
#include <algorithm> #include <algorithm>
#include <chrono> #include <chrono>
#include <stdexcept> #include <stdexcept>
#include <string> #include <string>
#include <utility> #include <utility>
#include <ggml-backend.h>
#include <llama.h>
#include "data_generation/llama_generator.h" #include "data_generation/llama_generator.h"
// Maximum batch size for decode operations. Capping the batch prevents // Maximum batch size for decode operations. Capping the batch prevents
@@ -20,42 +20,42 @@
static constexpr uint32_t kMaxBatchSize = 5000U; static constexpr uint32_t kMaxBatchSize = 5000U;
void LlamaGenerator::Load(const std::string& model_path) { void LlamaGenerator::Load(const std::string& model_path) {
context_.reset(); context_.reset();
model_.reset(); model_.reset();
// Specifically load dynamic ggml backends (like CUDA) that are provided // Specifically load dynamic ggml backends (like CUDA) that are provided
// externally before attempting to load a model. // externally before attempting to load a model.
ggml_backend_load_all(); ggml_backend_load_all();
llama_model_params model_params = llama_model_default_params(); llama_model_params model_params = llama_model_default_params();
model_params.n_gpu_layers = n_gpu_layers_; model_params.n_gpu_layers = n_gpu_layers_;
ModelHandle loaded_model( ModelHandle loaded_model(
llama_model_load_from_file(model_path.c_str(), model_params)); llama_model_load_from_file(model_path.c_str(), model_params));
if (!loaded_model) { if (!loaded_model) {
throw std::runtime_error( throw std::runtime_error(
"LlamaGenerator: failed to load model from path: " + model_path); "LlamaGenerator: failed to load model from path: " + model_path);
} }
llama_context_params context_params = llama_context_default_params(); llama_context_params context_params = llama_context_default_params();
context_params.n_ctx = n_ctx_; context_params.n_ctx = n_ctx_;
context_params.n_batch = std::min(n_ctx_, kMaxBatchSize); context_params.n_batch = std::min(n_ctx_, kMaxBatchSize);
ContextHandle loaded_context( ContextHandle loaded_context(
llama_init_from_model(loaded_model.get(), context_params)); llama_init_from_model(loaded_model.get(), context_params));
if (!loaded_context) { if (!loaded_context) {
throw std::runtime_error("LlamaGenerator: failed to create context"); throw std::runtime_error("LlamaGenerator: failed to create context");
} }
model_ = std::move(loaded_model); model_ = std::move(loaded_model);
context_ = std::move(loaded_context); context_ = std::move(loaded_context);
if (logger_) { if (logger_) {
logger_->Log({.level = LogLevel::Info, logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup, .phase = PipelinePhase::Startup,
.message = std::format("[LlamaGenerator] Loaded model: {} ", .message = std::format("[LlamaGenerator] Loaded model: {} ",
model_path)}); model_path)});
} }
} }