mirror of
https://github.com/aaronpo97/the-biergarten-app.git
synced 2026-10-15 10:04:44 +00:00
Formatted src/data_generation/llama
This commit is contained in:
Aaron Po
@@ -18,144 +18,144 @@
|
||||
|
||||
static std::string FormatLocalLanguageCodes(
|
||||
const std::vector<std::string>& codes) {
|
||||
if (codes.empty()) {
|
||||
return "Not provided";
|
||||
}
|
||||
if (codes.empty()) {
|
||||
return "Not provided";
|
||||
}
|
||||
|
||||
std::string formatted;
|
||||
for (const std::string& code : codes) {
|
||||
if (!formatted.empty()) {
|
||||
formatted += ", ";
|
||||
}
|
||||
formatted += code;
|
||||
}
|
||||
std::string formatted;
|
||||
for (const std::string& code : codes) {
|
||||
if (!formatted.empty()) {
|
||||
formatted += ", ";
|
||||
}
|
||||
formatted += code;
|
||||
}
|
||||
|
||||
return formatted;
|
||||
return formatted;
|
||||
}
|
||||
|
||||
static constexpr int kBreweryInitialMaxTokens = 2800;
|
||||
|
||||
BreweryResult LlamaGenerator::GenerateBrewery(
|
||||
const EnrichedCity& enriched_city) {
|
||||
const City& location = enriched_city.location;
|
||||
const std::string& region_context = enriched_city.region_context;
|
||||
/**
|
||||
* Preprocess and truncate region context to manageable size
|
||||
*/
|
||||
const std::string safe_region_context = PrepareRegionContext(region_context);
|
||||
const City& location = enriched_city.location;
|
||||
const std::string& region_context = enriched_city.region_context;
|
||||
/**
|
||||
* Preprocess and truncate region context to manageable size
|
||||
*/
|
||||
const std::string safe_region_context = PrepareRegionContext(region_context);
|
||||
|
||||
const std::string local_language_codes =
|
||||
FormatLocalLanguageCodes(location.local_languages);
|
||||
const std::string local_language_codes =
|
||||
FormatLocalLanguageCodes(location.local_languages);
|
||||
|
||||
const std::string country_suffix =
|
||||
location.country.empty() ? std::string{}
|
||||
: std::format(", {}", location.country);
|
||||
/**
|
||||
* Load brewery system prompt via the injected prompt directory.
|
||||
* The key "BREWERY_GENERATION" resolves to BREWERY_GENERATION.md inside
|
||||
* the configured --prompt-dir. Throws on missing or empty file.
|
||||
*/
|
||||
const std::string system_prompt =
|
||||
prompt_directory_->Load("BREWERY_GENERATION");
|
||||
const std::string country_suffix =
|
||||
location.country.empty() ? std::string{}
|
||||
: std::format(", {}", location.country);
|
||||
/**
|
||||
* Load brewery system prompt via the injected prompt directory.
|
||||
* The key "BREWERY_GENERATION" resolves to BREWERY_GENERATION.md inside
|
||||
* the configured --prompt-dir. Throws on missing or empty file.
|
||||
*/
|
||||
const std::string system_prompt =
|
||||
prompt_directory_->Load("BREWERY_GENERATION");
|
||||
|
||||
std::string user_prompt = std::format(
|
||||
"## CITY:\n{}\n\n## COUNTRY:\n{}\n\n## LOCAL LANGUAGE CODES:\n{}\n\n## "
|
||||
"CONTEXT:\n{}",
|
||||
location.city, location.country, local_language_codes,
|
||||
safe_region_context);
|
||||
std::string user_prompt = std::format(
|
||||
"## CITY:\n{}\n\n## COUNTRY:\n{}\n\n## LOCAL LANGUAGE CODES:\n{}\n\n## "
|
||||
"CONTEXT:\n{}",
|
||||
location.city, location.country, local_language_codes,
|
||||
safe_region_context);
|
||||
|
||||
/**
|
||||
* Store location context for retry prompts (without repeating full context)
|
||||
*/
|
||||
const std::string retry_location =
|
||||
std::format("City: {}{}\nLocal language codes: {}", location.city,
|
||||
country_suffix, local_language_codes);
|
||||
/**
|
||||
* Store location context for retry prompts (without repeating full context)
|
||||
*/
|
||||
const std::string retry_location =
|
||||
std::format("City: {}{}\nLocal language codes: {}", location.city,
|
||||
country_suffix, local_language_codes);
|
||||
|
||||
/**
|
||||
* RETRY LOOP with validation and error correction
|
||||
* Attempts to generate valid brewery data up to 3 times, with
|
||||
* feedback-based refinement
|
||||
*/
|
||||
constexpr int max_attempts = 3;
|
||||
std::string raw;
|
||||
std::string last_error;
|
||||
/**
|
||||
* RETRY LOOP with validation and error correction
|
||||
* Attempts to generate valid brewery data up to 3 times, with
|
||||
* feedback-based refinement
|
||||
*/
|
||||
constexpr int max_attempts = 3;
|
||||
std::string raw;
|
||||
std::string last_error;
|
||||
|
||||
// Token budget: too small risks truncating valid JSON mid-string.
|
||||
int max_tokens = kBreweryInitialMaxTokens;
|
||||
|
||||
// Limit output length to keep it concise and focused
|
||||
for (int attempt = 0; attempt < max_attempts; ++attempt) {
|
||||
// Generate brewery data from LLM
|
||||
raw = this->Infer(system_prompt, user_prompt, max_tokens,
|
||||
kBreweryJsonGrammar);
|
||||
if (logger_) {
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Debug,
|
||||
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
||||
.message = std::format("LlamaGenerator: raw output (attempt {}): {}",
|
||||
attempt + 1, raw)});
|
||||
}
|
||||
|
||||
// Validate output: parse JSON and check required fields
|
||||
|
||||
BreweryResult brewery;
|
||||
const std::optional<std::string> validation_error =
|
||||
ValidateBreweryJson(raw, brewery);
|
||||
|
||||
if (!validation_error.has_value()) {
|
||||
// Success: return parsed brewery data
|
||||
// Token budget: too small risks truncating valid JSON mid-string.
|
||||
int max_tokens = kBreweryInitialMaxTokens;
|
||||
|
||||
// Limit output length to keep it concise and focused
|
||||
for (int attempt = 0; attempt < max_attempts; ++attempt) {
|
||||
// Generate brewery data from LLM
|
||||
raw = this->Infer(system_prompt, user_prompt, max_tokens,
|
||||
kBreweryJsonGrammar);
|
||||
if (logger_) {
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
||||
.message = std::format("LlamaGenerator: successfully generated "
|
||||
"brewery data on attempt {}",
|
||||
attempt + 1)});
|
||||
logger_->Log({.level = LogLevel::Debug,
|
||||
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
||||
.message = std::format(
|
||||
"LlamaGenerator: raw output (attempt {}): {}",
|
||||
attempt + 1, raw)});
|
||||
}
|
||||
|
||||
return brewery;
|
||||
}
|
||||
// Validate output: parse JSON and check required fields
|
||||
|
||||
// Validation failed: log error and prepare corrective feedback
|
||||
BreweryResult brewery;
|
||||
const std::optional<std::string> validation_error =
|
||||
ValidateBreweryJson(raw, brewery);
|
||||
|
||||
last_error = *validation_error;
|
||||
if (logger_) {
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
||||
.message = std::format(
|
||||
"LlamaGenerator: malformed brewery JSON (attempt {}): {}",
|
||||
attempt + 1, *validation_error)});
|
||||
}
|
||||
if (!validation_error.has_value()) {
|
||||
// Success: return parsed brewery data
|
||||
|
||||
// Update prompt with error details to guide LLM toward correct output.
|
||||
user_prompt = std::format(
|
||||
"Your previous response was invalid. Error: {}\nReturn the thought "
|
||||
"process before the JSON if needed, then return ONLY valid JSON "
|
||||
"with "
|
||||
"exactly these keys, in this exact order: {{\"name_en\": "
|
||||
"\"<English "
|
||||
"brewery name>\", \"description_en\": \"<English single-paragraph "
|
||||
"description>\", \"name_local\": \"<local-language brewery "
|
||||
"name>\", "
|
||||
"\"description_local\": \"<local-language single-paragraph "
|
||||
"description>\"}}.\nDo not include markdown, comments, extra keys, "
|
||||
"or "
|
||||
"literal placeholder values.\n\nKeep the JSON strings concise "
|
||||
"enough "
|
||||
"to fit within the token budget.\n\n{}",
|
||||
*validation_error, retry_location);
|
||||
}
|
||||
if (logger_) {
|
||||
logger_->Log({.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
||||
.message = std::format(
|
||||
"LlamaGenerator: successfully generated "
|
||||
"brewery data on attempt {}",
|
||||
attempt + 1)});
|
||||
}
|
||||
|
||||
// All retry attempts exhausted: log failure and throw exception
|
||||
if (logger_) {
|
||||
logger_->Log({.level = LogLevel::Error,
|
||||
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
||||
.message = std::format(
|
||||
"LlamaGenerator: malformed brewery "
|
||||
"response after {} attempts: {}",
|
||||
max_attempts, last_error.empty() ? raw : last_error)});
|
||||
}
|
||||
throw std::runtime_error("LlamaGenerator: malformed brewery response");
|
||||
return brewery;
|
||||
}
|
||||
|
||||
// Validation failed: log error and prepare corrective feedback
|
||||
|
||||
last_error = *validation_error;
|
||||
if (logger_) {
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
||||
.message = std::format(
|
||||
"LlamaGenerator: malformed brewery JSON (attempt {}): {}",
|
||||
attempt + 1, *validation_error)});
|
||||
}
|
||||
|
||||
// Update prompt with error details to guide LLM toward correct output.
|
||||
user_prompt = std::format(
|
||||
"Your previous response was invalid. Error: {}\nReturn the thought "
|
||||
"process before the JSON if needed, then return ONLY valid JSON "
|
||||
"with "
|
||||
"exactly these keys, in this exact order: {{\"name_en\": "
|
||||
"\"<English "
|
||||
"brewery name>\", \"description_en\": \"<English single-paragraph "
|
||||
"description>\", \"name_local\": \"<local-language brewery "
|
||||
"name>\", "
|
||||
"\"description_local\": \"<local-language single-paragraph "
|
||||
"description>\"}}.\nDo not include markdown, comments, extra keys, "
|
||||
"or "
|
||||
"literal placeholder values.\n\nKeep the JSON strings concise "
|
||||
"enough "
|
||||
"to fit within the token budget.\n\n{}",
|
||||
*validation_error, retry_location);
|
||||
}
|
||||
|
||||
// All retry attempts exhausted: log failure and throw exception
|
||||
if (logger_) {
|
||||
logger_->Log({.level = LogLevel::Error,
|
||||
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
||||
.message = std::format(
|
||||
"LlamaGenerator: malformed brewery "
|
||||
"response after {} attempts: {}",
|
||||
max_attempts, last_error.empty() ? raw : last_error)});
|
||||
}
|
||||
throw std::runtime_error("LlamaGenerator: malformed brewery response");
|
||||
}
|
||||
|
||||
@@ -19,97 +19,99 @@ static constexpr int kUserInitialMaxTokens = 1200;
|
||||
UserResult LlamaGenerator::GenerateUser(const EnrichedCity& city,
|
||||
const UserPersona& persona,
|
||||
const Name& name) {
|
||||
std::string style_affinities;
|
||||
for (const std::string& style : persona.style_affinities) {
|
||||
if (!style_affinities.empty()) {
|
||||
style_affinities += ", ";
|
||||
}
|
||||
style_affinities += style;
|
||||
}
|
||||
std::string style_affinities;
|
||||
for (const std::string& style : persona.style_affinities) {
|
||||
if (!style_affinities.empty()) {
|
||||
style_affinities += ", ";
|
||||
}
|
||||
style_affinities += style;
|
||||
}
|
||||
|
||||
const std::string system_prompt = prompt_directory_->Load("USER_GENERATION");
|
||||
const std::string system_prompt = prompt_directory_->Load("USER_GENERATION");
|
||||
|
||||
std::string user_prompt = std::format(
|
||||
"## NAME:\n\n{} {}\n\n"
|
||||
"## GENDER:\n\n{}\n\n"
|
||||
"## CITY:\n\n{}\n\n"
|
||||
"## COUNTRY:\n\n{}\n\n"
|
||||
"## PERSONA:\n\n{}\n\n"
|
||||
"## PERSONA DESCRIPTION:\n\n{}\n\n"
|
||||
"## STYLE AFFINITIES:\n\n{}",
|
||||
name.first_name, name.last_name, name.gender, city.location.city,
|
||||
city.location.country, persona.name, persona.description,
|
||||
style_affinities);
|
||||
std::string user_prompt = std::format(
|
||||
"## NAME:\n\n{} {}\n\n"
|
||||
"## GENDER:\n\n{}\n\n"
|
||||
"## CITY:\n\n{}\n\n"
|
||||
"## COUNTRY:\n\n{}\n\n"
|
||||
"## PERSONA:\n\n{}\n\n"
|
||||
"## PERSONA DESCRIPTION:\n\n{}\n\n"
|
||||
"## STYLE AFFINITIES:\n\n{}",
|
||||
name.first_name, name.last_name, name.gender, city.location.city,
|
||||
city.location.country, persona.name, persona.description,
|
||||
style_affinities);
|
||||
|
||||
const std::string retry_context = std::format(
|
||||
"Name: {} {}\nCity: {}, {}\nPersona: {}", name.first_name, name.last_name,
|
||||
city.location.city, city.location.country, persona.name);
|
||||
const std::string retry_context = std::format(
|
||||
"Name: {} {}\nCity: {}, {}\nPersona: {}", name.first_name,
|
||||
name.last_name, city.location.city, city.location.country, persona.name);
|
||||
|
||||
constexpr int max_attempts = 3;
|
||||
std::string raw;
|
||||
std::string last_error;
|
||||
int max_tokens = kUserInitialMaxTokens;
|
||||
constexpr int max_attempts = 3;
|
||||
std::string raw;
|
||||
std::string last_error;
|
||||
int max_tokens = kUserInitialMaxTokens;
|
||||
|
||||
for (int attempt = 0; attempt < max_attempts; ++attempt) {
|
||||
raw = this->Infer(system_prompt, user_prompt, max_tokens, kUserJsonGrammar);
|
||||
if (logger_) {
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Debug,
|
||||
.phase = PipelinePhase::UserGeneration,
|
||||
.message = std::format("LlamaGenerator: raw output (attempt {}): {}",
|
||||
attempt + 1, raw)});
|
||||
}
|
||||
|
||||
UserResult user;
|
||||
const std::optional<std::string> validation_error =
|
||||
ValidateUserJson(raw, user);
|
||||
|
||||
if (!validation_error.has_value()) {
|
||||
for (int attempt = 0; attempt < max_attempts; ++attempt) {
|
||||
raw =
|
||||
this->Infer(system_prompt, user_prompt, max_tokens, kUserJsonGrammar);
|
||||
if (logger_) {
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::UserGeneration,
|
||||
.message = std::format("LlamaGenerator: successfully "
|
||||
"generated user data on attempt {}",
|
||||
attempt + 1)});
|
||||
logger_->Log({.level = LogLevel::Debug,
|
||||
.phase = PipelinePhase::UserGeneration,
|
||||
.message = std::format(
|
||||
"LlamaGenerator: raw output (attempt {}): {}",
|
||||
attempt + 1, raw)});
|
||||
}
|
||||
|
||||
user.first_name = name.first_name;
|
||||
user.last_name = name.last_name;
|
||||
user.gender = name.gender;
|
||||
return user;
|
||||
}
|
||||
UserResult user;
|
||||
const std::optional<std::string> validation_error =
|
||||
ValidateUserJson(raw, user);
|
||||
|
||||
last_error = *validation_error;
|
||||
if (logger_) {
|
||||
logger_->Log({.level = LogLevel::Warn,
|
||||
if (!validation_error.has_value()) {
|
||||
if (logger_) {
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::UserGeneration,
|
||||
.message = std::format("LlamaGenerator: successfully "
|
||||
"generated user data on attempt {}",
|
||||
attempt + 1)});
|
||||
}
|
||||
|
||||
user.first_name = name.first_name;
|
||||
user.last_name = name.last_name;
|
||||
user.gender = name.gender;
|
||||
return user;
|
||||
}
|
||||
|
||||
last_error = *validation_error;
|
||||
if (logger_) {
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::UserGeneration,
|
||||
.message = std::format(
|
||||
"LlamaGenerator: malformed user JSON (attempt {}): {}",
|
||||
attempt + 1, *validation_error)});
|
||||
}
|
||||
|
||||
user_prompt = std::format(
|
||||
"Your previous response was invalid. Error: {}\nReturn the thought "
|
||||
"process before the JSON if needed, then return ONLY valid JSON "
|
||||
"with "
|
||||
"exactly these keys, in this exact order: {{\"username\": "
|
||||
"\"<handle "
|
||||
"derived from the given name>\", \"bio\": \"<first-person bio "
|
||||
"grounded in the persona>\", \"activity_weight\": <number between "
|
||||
"0 "
|
||||
"and 1>}}.\nDo not include markdown, comments, extra keys, or "
|
||||
"literal placeholder values.\n\n{}",
|
||||
*validation_error, retry_context);
|
||||
}
|
||||
|
||||
if (logger_) {
|
||||
logger_->Log({.level = LogLevel::Error,
|
||||
.phase = PipelinePhase::UserGeneration,
|
||||
.message = std::format(
|
||||
"LlamaGenerator: malformed user JSON (attempt {}): {}",
|
||||
attempt + 1, *validation_error)});
|
||||
}
|
||||
|
||||
user_prompt = std::format(
|
||||
"Your previous response was invalid. Error: {}\nReturn the thought "
|
||||
"process before the JSON if needed, then return ONLY valid JSON "
|
||||
"with "
|
||||
"exactly these keys, in this exact order: {{\"username\": "
|
||||
"\"<handle "
|
||||
"derived from the given name>\", \"bio\": \"<first-person bio "
|
||||
"grounded in the persona>\", \"activity_weight\": <number between "
|
||||
"0 "
|
||||
"and 1>}}.\nDo not include markdown, comments, extra keys, or "
|
||||
"literal placeholder values.\n\n{}",
|
||||
*validation_error, retry_context);
|
||||
}
|
||||
|
||||
if (logger_) {
|
||||
logger_->Log({.level = LogLevel::Error,
|
||||
.phase = PipelinePhase::UserGeneration,
|
||||
.message = std::format(
|
||||
"LlamaGenerator: malformed user response "
|
||||
"after {} attempts: {}",
|
||||
max_attempts, last_error.empty() ? raw : last_error)});
|
||||
}
|
||||
throw std::runtime_error("LlamaGenerator: malformed user response");
|
||||
"LlamaGenerator: malformed user response "
|
||||
"after {} attempts: {}",
|
||||
max_attempts, last_error.empty() ? raw : last_error)});
|
||||
}
|
||||
throw std::runtime_error("LlamaGenerator: malformed user response");
|
||||
}
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
* parsing, token decoding, and JSON validation helpers for Llama modules.
|
||||
*/
|
||||
|
||||
#include <llama.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <boost/json.hpp>
|
||||
@@ -15,8 +17,6 @@
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
#include <llama.h>
|
||||
|
||||
#include "data_generation/llama_generator_helpers.h"
|
||||
|
||||
namespace {
|
||||
@@ -24,14 +24,14 @@ namespace {
|
||||
* String trimming: removes leading and trailing whitespace
|
||||
*/
|
||||
std::string Trim(std::string_view value) {
|
||||
constexpr std::string_view whitespace = " \t\n\r\f\v";
|
||||
const size_t first_index = value.find_first_not_of(whitespace);
|
||||
if (first_index == std::string_view::npos) {
|
||||
return {};
|
||||
}
|
||||
constexpr std::string_view whitespace = " \t\n\r\f\v";
|
||||
const size_t first_index = value.find_first_not_of(whitespace);
|
||||
if (first_index == std::string_view::npos) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const size_t last_index = value.find_last_not_of(whitespace);
|
||||
return std::string(value.substr(first_index, last_index - first_index + 1));
|
||||
const size_t last_index = value.find_last_not_of(whitespace);
|
||||
return std::string(value.substr(first_index, last_index - first_index + 1));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -39,26 +39,26 @@ std::string Trim(std::string_view value) {
|
||||
* spaces
|
||||
*/
|
||||
std::string CondenseWhitespace(std::string_view text) {
|
||||
std::string out;
|
||||
out.reserve(text.size());
|
||||
std::string out;
|
||||
out.reserve(text.size());
|
||||
|
||||
bool pending_space = false;
|
||||
for (const char chr : text) {
|
||||
if (std::isspace(static_cast<unsigned char>(chr)) != 0) {
|
||||
if (!out.empty()) {
|
||||
pending_space = true;
|
||||
bool pending_space = false;
|
||||
for (const char chr : text) {
|
||||
if (std::isspace(static_cast<unsigned char>(chr)) != 0) {
|
||||
if (!out.empty()) {
|
||||
pending_space = true;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (pending_space) {
|
||||
out.push_back(' ');
|
||||
pending_space = false;
|
||||
}
|
||||
out.push_back(chr);
|
||||
}
|
||||
if (pending_space) {
|
||||
out.push_back(' ');
|
||||
pending_space = false;
|
||||
}
|
||||
out.push_back(chr);
|
||||
}
|
||||
|
||||
return out;
|
||||
return out;
|
||||
}
|
||||
|
||||
// Guard against truncating in the first half of the string.
|
||||
@@ -69,30 +69,30 @@ constexpr size_t kTruncationGuardDivisor = 2;
|
||||
bool ReadRequiredTrimmedStringField(const boost::json::object& obj,
|
||||
std::string_view key, std::string& out,
|
||||
std::string* error_out) {
|
||||
const boost::json::value* field = obj.if_contains(key);
|
||||
if (field == nullptr || !field->is_string()) {
|
||||
return false;
|
||||
}
|
||||
const boost::json::value* field = obj.if_contains(key);
|
||||
if (field == nullptr || !field->is_string()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto& string_value = field->as_string();
|
||||
out = Trim(std::string_view(string_value.data(), string_value.size()));
|
||||
return !out.empty();
|
||||
const auto& string_value = field->as_string();
|
||||
out = Trim(std::string_view(string_value.data(), string_value.size()));
|
||||
return !out.empty();
|
||||
}
|
||||
|
||||
bool HasSchemaPlaceholder(std::span<std::string* const> values) {
|
||||
for (const std::string* value : values) {
|
||||
std::string lowered = *value;
|
||||
std::ranges::transform(lowered, lowered.begin(),
|
||||
[](const unsigned char character) {
|
||||
return static_cast<char>(std::tolower(character));
|
||||
});
|
||||
for (const std::string* value : values) {
|
||||
std::string lowered = *value;
|
||||
std::ranges::transform(
|
||||
lowered, lowered.begin(), [](const unsigned char character) {
|
||||
return static_cast<char>(std::tolower(character));
|
||||
});
|
||||
|
||||
if (lowered == "string") {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if (lowered == "string") {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
return false;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
@@ -102,166 +102,167 @@ bool HasSchemaPlaceholder(std::span<std::string* const> values) {
|
||||
*/
|
||||
std::string PrepareRegionContext(std::string_view region_context,
|
||||
const size_t max_chars) {
|
||||
std::string normalized = CondenseWhitespace(region_context);
|
||||
if (normalized.size() <= max_chars) {
|
||||
return normalized;
|
||||
}
|
||||
std::string normalized = CondenseWhitespace(region_context);
|
||||
if (normalized.size() <= max_chars) {
|
||||
return normalized;
|
||||
}
|
||||
|
||||
normalized.resize(max_chars);
|
||||
const size_t last_space = normalized.find_last_of(' ');
|
||||
if (last_space != std::string::npos &&
|
||||
last_space > max_chars / kTruncationGuardDivisor) {
|
||||
normalized.resize(last_space);
|
||||
}
|
||||
normalized.resize(max_chars);
|
||||
const size_t last_space = normalized.find_last_of(' ');
|
||||
if (last_space != std::string::npos &&
|
||||
last_space > max_chars / kTruncationGuardDivisor) {
|
||||
normalized.resize(last_space);
|
||||
}
|
||||
|
||||
normalized += "...";
|
||||
return normalized;
|
||||
normalized += "...";
|
||||
return normalized;
|
||||
}
|
||||
|
||||
void AppendTokenPiece(const llama_vocab* vocab, llama_token token,
|
||||
std::string& output) {
|
||||
constexpr size_t initial_buffer_size = 256;
|
||||
constexpr size_t initial_buffer_size = 256;
|
||||
|
||||
std::array<char, initial_buffer_size> buffer{};
|
||||
std::array<char, initial_buffer_size> buffer{};
|
||||
|
||||
// serialize the sampled token into UTF-8 bytes
|
||||
// serialize the sampled token into UTF-8 bytes
|
||||
|
||||
auto buffer_too_small = [](int32_t result) -> bool { return result < 0; };
|
||||
auto buffer_too_small = [](int32_t result) -> bool { return result < 0; };
|
||||
|
||||
int32_t bytes =
|
||||
llama_token_to_piece(vocab, token, buffer.data(), buffer.size(), 0, true);
|
||||
int32_t bytes = llama_token_to_piece(vocab, token, buffer.data(),
|
||||
buffer.size(), 0, true);
|
||||
|
||||
if (!buffer_too_small(bytes)) {
|
||||
// Append the decoded bytes from the stack buffer.
|
||||
output.append(buffer.data(), static_cast<size_t>(bytes));
|
||||
return;
|
||||
}
|
||||
if (!buffer_too_small(bytes)) {
|
||||
// Append the decoded bytes from the stack buffer.
|
||||
output.append(buffer.data(), static_cast<size_t>(bytes));
|
||||
return;
|
||||
}
|
||||
|
||||
const int32_t required_size = -bytes;
|
||||
std::vector<char> dynamic_buffer(static_cast<size_t>(required_size));
|
||||
const int32_t required_size = -bytes;
|
||||
std::vector<char> dynamic_buffer(static_cast<size_t>(required_size));
|
||||
|
||||
// Retry token decoding against the larger heap buffer.
|
||||
bytes = llama_token_to_piece(vocab, token, dynamic_buffer.data(),
|
||||
static_cast<int32_t>(dynamic_buffer.size()), 0,
|
||||
true);
|
||||
// Retry token decoding against the larger heap buffer.
|
||||
bytes = llama_token_to_piece(vocab, token, dynamic_buffer.data(),
|
||||
static_cast<int32_t>(dynamic_buffer.size()), 0,
|
||||
true);
|
||||
|
||||
if (!buffer_too_small(bytes)) {
|
||||
output.append(dynamic_buffer.data(), static_cast<size_t>(bytes));
|
||||
return;
|
||||
}
|
||||
if (!buffer_too_small(bytes)) {
|
||||
output.append(dynamic_buffer.data(), static_cast<size_t>(bytes));
|
||||
return;
|
||||
}
|
||||
|
||||
throw std::runtime_error(
|
||||
"LlamaGenerator: failed to decode sampled token piece");
|
||||
throw std::runtime_error(
|
||||
"LlamaGenerator: failed to decode sampled token piece");
|
||||
}
|
||||
|
||||
std::optional<std::string> ValidateBreweryJson(const std::string& raw,
|
||||
BreweryResult& brewery_out) {
|
||||
boost::system::error_code error_code;
|
||||
const std::string_view raw_view(raw);
|
||||
const size_t opening_brace = raw_view.find('{');
|
||||
if (opening_brace == std::string_view::npos) {
|
||||
return "JSON parse error: missing opening brace '{'";
|
||||
}
|
||||
boost::system::error_code error_code;
|
||||
const std::string_view raw_view(raw);
|
||||
const size_t opening_brace = raw_view.find('{');
|
||||
if (opening_brace == std::string_view::npos) {
|
||||
return "JSON parse error: missing opening brace '{'";
|
||||
}
|
||||
|
||||
const std::string_view json_payload = raw_view.substr(opening_brace);
|
||||
boost::json::value json_value = boost::json::parse(json_payload, error_code);
|
||||
if (error_code) {
|
||||
return "JSON parse error: " + error_code.message();
|
||||
}
|
||||
const std::string_view json_payload = raw_view.substr(opening_brace);
|
||||
boost::json::value json_value = boost::json::parse(json_payload, error_code);
|
||||
if (error_code) {
|
||||
return "JSON parse error: " + error_code.message();
|
||||
}
|
||||
|
||||
if (!json_value.is_object()) {
|
||||
return "JSON root must be an object";
|
||||
}
|
||||
if (!json_value.is_object()) {
|
||||
return "JSON root must be an object";
|
||||
}
|
||||
|
||||
const auto& obj = json_value.get_object();
|
||||
if (obj.size() != 4) {
|
||||
return "JSON object must contain exactly four keys";
|
||||
}
|
||||
const auto& obj = json_value.get_object();
|
||||
if (obj.size() != 4) {
|
||||
return "JSON object must contain exactly four keys";
|
||||
}
|
||||
|
||||
std::string validation_error;
|
||||
if (!ReadRequiredTrimmedStringField(obj, "name_en", brewery_out.name_en,
|
||||
&validation_error)) {
|
||||
return validation_error;
|
||||
}
|
||||
std::string validation_error;
|
||||
if (!ReadRequiredTrimmedStringField(obj, "name_en", brewery_out.name_en,
|
||||
&validation_error)) {
|
||||
return validation_error;
|
||||
}
|
||||
|
||||
if (!ReadRequiredTrimmedStringField(obj, "description_en",
|
||||
brewery_out.description_en,
|
||||
&validation_error)) {
|
||||
return validation_error;
|
||||
}
|
||||
if (!ReadRequiredTrimmedStringField(obj, "description_en",
|
||||
brewery_out.description_en,
|
||||
&validation_error)) {
|
||||
return validation_error;
|
||||
}
|
||||
|
||||
if (!ReadRequiredTrimmedStringField(obj, "name_local", brewery_out.name_local,
|
||||
&validation_error)) {
|
||||
return validation_error;
|
||||
}
|
||||
if (!ReadRequiredTrimmedStringField(
|
||||
obj, "name_local", brewery_out.name_local, &validation_error)) {
|
||||
return validation_error;
|
||||
}
|
||||
|
||||
if (!ReadRequiredTrimmedStringField(obj, "description_local",
|
||||
brewery_out.description_local,
|
||||
&validation_error)) {
|
||||
return validation_error;
|
||||
}
|
||||
if (!ReadRequiredTrimmedStringField(obj, "description_local",
|
||||
brewery_out.description_local,
|
||||
&validation_error)) {
|
||||
return validation_error;
|
||||
}
|
||||
|
||||
const std::array schema_placeholders = {
|
||||
&brewery_out.name_en, &brewery_out.description_en,
|
||||
&brewery_out.name_local, &brewery_out.description_local};
|
||||
if (HasSchemaPlaceholder(schema_placeholders)) {
|
||||
return "JSON appears to be a schema placeholder, not content";
|
||||
}
|
||||
const std::array schema_placeholders = {
|
||||
&brewery_out.name_en, &brewery_out.description_en,
|
||||
&brewery_out.name_local, &brewery_out.description_local};
|
||||
if (HasSchemaPlaceholder(schema_placeholders)) {
|
||||
return "JSON appears to be a schema placeholder, not content";
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::optional<std::string> ValidateUserJson(const std::string& raw,
|
||||
UserResult& user_out) {
|
||||
boost::system::error_code error_code;
|
||||
const std::string_view raw_view(raw);
|
||||
const size_t opening_brace = raw_view.find('{');
|
||||
if (opening_brace == std::string_view::npos) {
|
||||
return "JSON parse error: missing opening brace '{'";
|
||||
}
|
||||
boost::system::error_code error_code;
|
||||
const std::string_view raw_view(raw);
|
||||
const size_t opening_brace = raw_view.find('{');
|
||||
if (opening_brace == std::string_view::npos) {
|
||||
return "JSON parse error: missing opening brace '{'";
|
||||
}
|
||||
|
||||
const std::string_view json_payload = raw_view.substr(opening_brace);
|
||||
boost::json::value json_value = boost::json::parse(json_payload, error_code);
|
||||
if (error_code) {
|
||||
return "JSON parse error: " + error_code.message();
|
||||
}
|
||||
const std::string_view json_payload = raw_view.substr(opening_brace);
|
||||
boost::json::value json_value = boost::json::parse(json_payload, error_code);
|
||||
if (error_code) {
|
||||
return "JSON parse error: " + error_code.message();
|
||||
}
|
||||
|
||||
if (!json_value.is_object()) {
|
||||
return "JSON root must be an object";
|
||||
}
|
||||
if (!json_value.is_object()) {
|
||||
return "JSON root must be an object";
|
||||
}
|
||||
|
||||
const auto& obj = json_value.get_object();
|
||||
if (obj.size() != 3) {
|
||||
return "JSON object must contain exactly three keys";
|
||||
}
|
||||
const auto& obj = json_value.get_object();
|
||||
if (obj.size() != 3) {
|
||||
return "JSON object must contain exactly three keys";
|
||||
}
|
||||
|
||||
std::string validation_error;
|
||||
if (!ReadRequiredTrimmedStringField(obj, "username", user_out.username,
|
||||
&validation_error)) {
|
||||
return validation_error;
|
||||
}
|
||||
std::string validation_error;
|
||||
if (!ReadRequiredTrimmedStringField(obj, "username", user_out.username,
|
||||
&validation_error)) {
|
||||
return validation_error;
|
||||
}
|
||||
|
||||
if (!ReadRequiredTrimmedStringField(obj, "bio", user_out.bio,
|
||||
&validation_error)) {
|
||||
return validation_error;
|
||||
}
|
||||
if (!ReadRequiredTrimmedStringField(obj, "bio", user_out.bio,
|
||||
&validation_error)) {
|
||||
return validation_error;
|
||||
}
|
||||
|
||||
const boost::json::value* activity_weight_field =
|
||||
obj.if_contains("activity_weight");
|
||||
if (activity_weight_field == nullptr || !activity_weight_field->is_number()) {
|
||||
return "Missing or invalid numeric field: activity_weight";
|
||||
}
|
||||
const boost::json::value* activity_weight_field =
|
||||
obj.if_contains("activity_weight");
|
||||
if (activity_weight_field == nullptr ||
|
||||
!activity_weight_field->is_number()) {
|
||||
return "Missing or invalid numeric field: activity_weight";
|
||||
}
|
||||
|
||||
const double activity_weight = activity_weight_field->to_number<double>();
|
||||
if (activity_weight < 0.0 || activity_weight > 1.0) {
|
||||
return "activity_weight must be between 0 and 1";
|
||||
}
|
||||
user_out.activity_weight = static_cast<float>(activity_weight);
|
||||
const double activity_weight = activity_weight_field->to_number<double>();
|
||||
if (activity_weight < 0.0 || activity_weight > 1.0) {
|
||||
return "activity_weight must be between 0 and 1";
|
||||
}
|
||||
user_out.activity_weight = static_cast<float>(activity_weight);
|
||||
|
||||
const std::array schema_placeholders = {&user_out.username, &user_out.bio};
|
||||
if (HasSchemaPlaceholder(schema_placeholders)) {
|
||||
return "JSON appears to be a schema placeholder, not content";
|
||||
}
|
||||
const std::array schema_placeholders = {&user_out.username, &user_out.bio};
|
||||
if (HasSchemaPlaceholder(schema_placeholders)) {
|
||||
return "JSON appears to be a schema placeholder, not content";
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
* output tokens back to text for system+user chat prompts.
|
||||
*/
|
||||
|
||||
#include <llama.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <format>
|
||||
@@ -14,8 +16,6 @@
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
#include <llama.h>
|
||||
|
||||
#include "data_generation/llama_generator.h"
|
||||
#include "data_generation/llama_generator_helpers.h"
|
||||
|
||||
@@ -30,48 +30,49 @@ using SamplerHandle =
|
||||
std::unique_ptr<llama_sampler, decltype(&llama_sampler_free)>;
|
||||
|
||||
struct SamplerConfig {
|
||||
float temperature;
|
||||
uint32_t top_k;
|
||||
float top_p;
|
||||
uint32_t seed;
|
||||
float temperature;
|
||||
uint32_t top_k;
|
||||
float top_p;
|
||||
uint32_t seed;
|
||||
};
|
||||
|
||||
SamplerHandle MakeSamplerChain(const llama_vocab* vocab,
|
||||
const SamplerConfig& config,
|
||||
std::string_view grammar) {
|
||||
const llama_sampler_chain_params sampler_params =
|
||||
llama_sampler_chain_default_params();
|
||||
const llama_sampler_chain_params sampler_params =
|
||||
llama_sampler_chain_default_params();
|
||||
|
||||
SamplerHandle chain(llama_sampler_chain_init(sampler_params),
|
||||
llama_sampler_free);
|
||||
if (!chain) {
|
||||
throw std::runtime_error("LlamaGenerator: failed to initialize sampler");
|
||||
}
|
||||
SamplerHandle chain(llama_sampler_chain_init(sampler_params),
|
||||
llama_sampler_free);
|
||||
if (!chain) {
|
||||
throw std::runtime_error("LlamaGenerator: failed to initialize sampler");
|
||||
}
|
||||
|
||||
auto add_sampler = [&](llama_sampler* sampler, const char* error_message) {
|
||||
if (sampler == nullptr) {
|
||||
throw std::runtime_error(error_message);
|
||||
}
|
||||
auto add_sampler = [&](llama_sampler* sampler, const char* error_message) {
|
||||
if (sampler == nullptr) {
|
||||
throw std::runtime_error(error_message);
|
||||
}
|
||||
|
||||
llama_sampler_chain_add(chain.get(), sampler);
|
||||
};
|
||||
llama_sampler_chain_add(chain.get(), sampler);
|
||||
};
|
||||
|
||||
if (!grammar.empty()) {
|
||||
const std::string grammar_text(grammar);
|
||||
add_sampler(llama_sampler_init_grammar(vocab, grammar_text.c_str(), "root"),
|
||||
"LlamaGenerator: failed to initialize grammar sampler");
|
||||
}
|
||||
if (!grammar.empty()) {
|
||||
const std::string grammar_text(grammar);
|
||||
add_sampler(
|
||||
llama_sampler_init_grammar(vocab, grammar_text.c_str(), "root"),
|
||||
"LlamaGenerator: failed to initialize grammar sampler");
|
||||
}
|
||||
|
||||
add_sampler(llama_sampler_init_temp(config.temperature),
|
||||
"LlamaGenerator: failed to initialize temperature sampler");
|
||||
add_sampler(llama_sampler_init_top_k(static_cast<int32_t>(config.top_k)),
|
||||
"LlamaGenerator: failed to initialize top-k sampler");
|
||||
add_sampler(llama_sampler_init_top_p(config.top_p, kTopPMinKeep),
|
||||
"LlamaGenerator: failed to initialize top-p sampler");
|
||||
add_sampler(llama_sampler_init_dist(config.seed),
|
||||
"LlamaGenerator: failed to initialize distribution sampler");
|
||||
add_sampler(llama_sampler_init_temp(config.temperature),
|
||||
"LlamaGenerator: failed to initialize temperature sampler");
|
||||
add_sampler(llama_sampler_init_top_k(static_cast<int32_t>(config.top_k)),
|
||||
"LlamaGenerator: failed to initialize top-k sampler");
|
||||
add_sampler(llama_sampler_init_top_p(config.top_p, kTopPMinKeep),
|
||||
"LlamaGenerator: failed to initialize top-p sampler");
|
||||
add_sampler(llama_sampler_init_dist(config.seed),
|
||||
"LlamaGenerator: failed to initialize distribution sampler");
|
||||
|
||||
return chain;
|
||||
return chain;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
@@ -80,163 +81,164 @@ std::string LlamaGenerator::Infer(const std::string& system_prompt,
|
||||
const std::string& prompt,
|
||||
const int max_tokens,
|
||||
std::string_view grammar) {
|
||||
return InferFormatted(prompt_formatter_->Format(system_prompt, prompt),
|
||||
max_tokens, grammar);
|
||||
return InferFormatted(prompt_formatter_->Format(system_prompt, prompt),
|
||||
max_tokens, grammar);
|
||||
}
|
||||
|
||||
std::string LlamaGenerator::InferFormatted(const std::string& formatted_prompt,
|
||||
const int max_tokens,
|
||||
std::string_view grammar) {
|
||||
/**
|
||||
* Validate that model and context are loaded
|
||||
*/
|
||||
if (!model_ || !context_) {
|
||||
throw std::runtime_error("LlamaGenerator: model not loaded");
|
||||
}
|
||||
/**
|
||||
* Validate that model and context are loaded
|
||||
*/
|
||||
if (!model_ || !context_) {
|
||||
throw std::runtime_error("LlamaGenerator: model not loaded");
|
||||
}
|
||||
|
||||
/**
|
||||
* Get vocabulary for tokenization and token-to-text conversion
|
||||
*/
|
||||
const llama_vocab* vocab = llama_model_get_vocab(model_.get());
|
||||
if (vocab == nullptr) {
|
||||
throw std::runtime_error("LlamaGenerator: vocab unavailable");
|
||||
}
|
||||
/**
|
||||
* Get vocabulary for tokenization and token-to-text conversion
|
||||
*/
|
||||
const llama_vocab* vocab = llama_model_get_vocab(model_.get());
|
||||
if (vocab == nullptr) {
|
||||
throw std::runtime_error("LlamaGenerator: vocab unavailable");
|
||||
}
|
||||
|
||||
const SamplerConfig sampler_config{
|
||||
.temperature = sampling_temperature_,
|
||||
.top_k = sampling_top_k_,
|
||||
.top_p = sampling_top_p_,
|
||||
.seed = static_cast<uint32_t>(rng_()),
|
||||
};
|
||||
const auto sampler = MakeSamplerChain(vocab, sampler_config, grammar);
|
||||
const SamplerConfig sampler_config{
|
||||
.temperature = sampling_temperature_,
|
||||
.top_k = sampling_top_k_,
|
||||
.top_p = sampling_top_p_,
|
||||
.seed = static_cast<uint32_t>(rng_()),
|
||||
};
|
||||
const auto sampler = MakeSamplerChain(vocab, sampler_config, grammar);
|
||||
|
||||
/**
|
||||
* Clear KV cache to ensure clean inference state (no residual context)
|
||||
*/
|
||||
llama_memory_clear(llama_get_memory(context_.get()), true);
|
||||
/**
|
||||
* Clear KV cache to ensure clean inference state (no residual context)
|
||||
*/
|
||||
llama_memory_clear(llama_get_memory(context_.get()), true);
|
||||
|
||||
/**
|
||||
* TOKENIZATION PHASE
|
||||
* Convert text prompt into token IDs (integers) that the model understands
|
||||
*/
|
||||
std::vector<llama_token> prompt_tokens(formatted_prompt.size() +
|
||||
kPromptTokenSlack);
|
||||
/**
|
||||
* TOKENIZATION PHASE
|
||||
* Convert text prompt into token IDs (integers) that the model understands
|
||||
*/
|
||||
std::vector<llama_token> prompt_tokens(formatted_prompt.size() +
|
||||
kPromptTokenSlack);
|
||||
|
||||
int32_t token_count = llama_tokenize(
|
||||
vocab, formatted_prompt.c_str(),
|
||||
static_cast<int32_t>(formatted_prompt.size()), prompt_tokens.data(),
|
||||
static_cast<int32_t>(prompt_tokens.size()), true, true);
|
||||
int32_t token_count = llama_tokenize(
|
||||
vocab, formatted_prompt.c_str(),
|
||||
static_cast<int32_t>(formatted_prompt.size()), prompt_tokens.data(),
|
||||
static_cast<int32_t>(prompt_tokens.size()), true, true);
|
||||
|
||||
/**
|
||||
* If buffer too small, negative return indicates required size
|
||||
*/
|
||||
if (token_count < 0) {
|
||||
prompt_tokens.resize(static_cast<size_t>(-token_count));
|
||||
/**
|
||||
* If buffer too small, negative return indicates required size
|
||||
*/
|
||||
if (token_count < 0) {
|
||||
prompt_tokens.resize(static_cast<size_t>(-token_count));
|
||||
|
||||
token_count = llama_tokenize(
|
||||
vocab, formatted_prompt.c_str(),
|
||||
static_cast<int32_t>(formatted_prompt.size()), prompt_tokens.data(),
|
||||
static_cast<int32_t>(prompt_tokens.size()), true, true);
|
||||
}
|
||||
token_count = llama_tokenize(
|
||||
vocab, formatted_prompt.c_str(),
|
||||
static_cast<int32_t>(formatted_prompt.size()), prompt_tokens.data(),
|
||||
static_cast<int32_t>(prompt_tokens.size()), true, true);
|
||||
}
|
||||
|
||||
if (token_count < 0) {
|
||||
throw std::runtime_error("LlamaGenerator: prompt tokenization failed");
|
||||
}
|
||||
if (token_count < 0) {
|
||||
throw std::runtime_error("LlamaGenerator: prompt tokenization failed");
|
||||
}
|
||||
|
||||
/**
|
||||
* CONTEXT SIZE VALIDATION
|
||||
* Validate and compute effective token budgets based on context window
|
||||
* constraints
|
||||
*/
|
||||
const auto n_ctx = static_cast<int32_t>(llama_n_ctx(context_.get()));
|
||||
const auto n_batch = static_cast<int32_t>(llama_n_batch(context_.get()));
|
||||
if (n_ctx <= 1 || n_batch <= 0) {
|
||||
throw std::runtime_error("LlamaGenerator: invalid context or batch size");
|
||||
}
|
||||
/**
|
||||
* CONTEXT SIZE VALIDATION
|
||||
* Validate and compute effective token budgets based on context window
|
||||
* constraints
|
||||
*/
|
||||
const auto n_ctx = static_cast<int32_t>(llama_n_ctx(context_.get()));
|
||||
const auto n_batch = static_cast<int32_t>(llama_n_batch(context_.get()));
|
||||
if (n_ctx <= 1 || n_batch <= 0) {
|
||||
throw std::runtime_error("LlamaGenerator: invalid context or batch size");
|
||||
}
|
||||
|
||||
/**
|
||||
* Clamp generation limit to available context window, reserve space for
|
||||
* output
|
||||
*/
|
||||
const int32_t effective_max_tokens =
|
||||
std::max(1, std::min(max_tokens, n_ctx - 1));
|
||||
/**
|
||||
* Clamp generation limit to available context window, reserve space for
|
||||
* output
|
||||
*/
|
||||
const int32_t effective_max_tokens =
|
||||
std::max(1, std::min(max_tokens, n_ctx - 1));
|
||||
|
||||
/**
|
||||
* Prompt can use remaining context after reserving space for generation
|
||||
*/
|
||||
int32_t prompt_budget = std::min(n_batch, n_ctx - effective_max_tokens);
|
||||
prompt_budget = std::max<int32_t>(1, prompt_budget);
|
||||
/**
|
||||
* Prompt can use remaining context after reserving space for generation
|
||||
*/
|
||||
int32_t prompt_budget = std::min(n_batch, n_ctx - effective_max_tokens);
|
||||
prompt_budget = std::max<int32_t>(1, prompt_budget);
|
||||
|
||||
/**
|
||||
* Truncate prompt if necessary to fit within constraints
|
||||
*/
|
||||
prompt_tokens.resize(static_cast<size_t>(token_count));
|
||||
if (token_count > prompt_budget) {
|
||||
if (logger_) {
|
||||
logger_->Log({.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
||||
.message = std::format(
|
||||
"LlamaGenerator: prompt too long ({} tokens), "
|
||||
"truncating to {} tokens to fit n_batch/n_ctx limits",
|
||||
token_count, prompt_budget)});
|
||||
}
|
||||
prompt_tokens.resize(static_cast<size_t>(prompt_budget));
|
||||
token_count = prompt_budget;
|
||||
}
|
||||
/**
|
||||
* Truncate prompt if necessary to fit within constraints
|
||||
*/
|
||||
prompt_tokens.resize(static_cast<size_t>(token_count));
|
||||
if (token_count > prompt_budget) {
|
||||
if (logger_) {
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
||||
.message = std::format(
|
||||
"LlamaGenerator: prompt too long ({} tokens), "
|
||||
"truncating to {} tokens to fit n_batch/n_ctx limits",
|
||||
token_count, prompt_budget)});
|
||||
}
|
||||
prompt_tokens.resize(static_cast<size_t>(prompt_budget));
|
||||
token_count = prompt_budget;
|
||||
}
|
||||
|
||||
/**
|
||||
* PROMPT PROCESSING PHASE
|
||||
* Create a batch containing all prompt tokens and feed through the model
|
||||
* This computes internal representations and fills the KV cache
|
||||
*/
|
||||
const llama_batch prompt_batch = llama_batch_get_one(
|
||||
prompt_tokens.data(), static_cast<int32_t>(prompt_tokens.size()));
|
||||
if (llama_decode(context_.get(), prompt_batch) != 0) {
|
||||
throw std::runtime_error("LlamaGenerator: prompt decode failed");
|
||||
}
|
||||
/**
|
||||
* PROMPT PROCESSING PHASE
|
||||
* Create a batch containing all prompt tokens and feed through the model
|
||||
* This computes internal representations and fills the KV cache
|
||||
*/
|
||||
const llama_batch prompt_batch = llama_batch_get_one(
|
||||
prompt_tokens.data(), static_cast<int32_t>(prompt_tokens.size()));
|
||||
if (llama_decode(context_.get(), prompt_batch) != 0) {
|
||||
throw std::runtime_error("LlamaGenerator: prompt decode failed");
|
||||
}
|
||||
|
||||
/**
|
||||
* TOKEN GENERATION LOOP
|
||||
* Iteratively generate tokens one at a time until max_tokens or
|
||||
* end-of-sequence
|
||||
*/
|
||||
std::vector<llama_token> generated_tokens;
|
||||
generated_tokens.reserve(static_cast<size_t>(effective_max_tokens));
|
||||
/**
|
||||
* TOKEN GENERATION LOOP
|
||||
* Iteratively generate tokens one at a time until max_tokens or
|
||||
* end-of-sequence
|
||||
*/
|
||||
std::vector<llama_token> generated_tokens;
|
||||
generated_tokens.reserve(static_cast<size_t>(effective_max_tokens));
|
||||
|
||||
for (int i = 0; i < effective_max_tokens; ++i) {
|
||||
/**
|
||||
* Sample next token using configured sampler chain and model logits
|
||||
* Index -1 means use the last output position from previous batch
|
||||
*/
|
||||
const llama_token next =
|
||||
llama_sampler_sample(sampler.get(), context_.get(), -1);
|
||||
/**
|
||||
* Stop if model predicts end-of-generation token (EOS/EOT)
|
||||
*/
|
||||
if (llama_vocab_is_eog(vocab, next)) {
|
||||
break;
|
||||
}
|
||||
generated_tokens.push_back(next);
|
||||
/**
|
||||
* Feed the sampled token back into model for next iteration
|
||||
* (autoregressive)
|
||||
*/
|
||||
llama_token decode_token = next;
|
||||
const llama_batch one_token_batch = llama_batch_get_one(&decode_token, 1);
|
||||
if (llama_decode(context_.get(), one_token_batch) != 0) {
|
||||
throw std::runtime_error(
|
||||
"LlamaGenerator: decode failed during generation");
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < effective_max_tokens; ++i) {
|
||||
/**
|
||||
* Sample next token using configured sampler chain and model logits
|
||||
* Index -1 means use the last output position from previous batch
|
||||
*/
|
||||
const llama_token next =
|
||||
llama_sampler_sample(sampler.get(), context_.get(), -1);
|
||||
/**
|
||||
* Stop if model predicts end-of-generation token (EOS/EOT)
|
||||
*/
|
||||
if (llama_vocab_is_eog(vocab, next)) {
|
||||
break;
|
||||
}
|
||||
generated_tokens.push_back(next);
|
||||
/**
|
||||
* Feed the sampled token back into model for next iteration
|
||||
* (autoregressive)
|
||||
*/
|
||||
llama_token decode_token = next;
|
||||
const llama_batch one_token_batch = llama_batch_get_one(&decode_token, 1);
|
||||
if (llama_decode(context_.get(), one_token_batch) != 0) {
|
||||
throw std::runtime_error(
|
||||
"LlamaGenerator: decode failed during generation");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* DETOKENIZATION PHASE
|
||||
* Convert generated token IDs back to text using vocabulary
|
||||
*/
|
||||
std::string output;
|
||||
for (const llama_token token : generated_tokens) {
|
||||
AppendTokenPiece(vocab, token, output);
|
||||
}
|
||||
/**
|
||||
* DETOKENIZATION PHASE
|
||||
* Convert generated token IDs back to text using vocabulary
|
||||
*/
|
||||
std::string output;
|
||||
for (const llama_token token : generated_tokens) {
|
||||
AppendTokenPiece(vocab, token, output);
|
||||
}
|
||||
|
||||
return output;
|
||||
return output;
|
||||
}
|
||||
|
||||
@@ -5,30 +5,30 @@
|
||||
|
||||
#include "data_generation/llama_generator.h"
|
||||
|
||||
#include <llama.h>
|
||||
|
||||
#include <filesystem>
|
||||
#include <memory>
|
||||
#include <random>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
|
||||
#include <llama.h>
|
||||
|
||||
#include "data_model/models.h"
|
||||
|
||||
static constexpr uint32_t kMaxContextSize = 32768U;
|
||||
|
||||
void LlamaGenerator::ModelDeleter::operator()(
|
||||
llama_model* model) const noexcept {
|
||||
if (model != nullptr) {
|
||||
llama_model_free(model);
|
||||
}
|
||||
if (model != nullptr) {
|
||||
llama_model_free(model);
|
||||
}
|
||||
}
|
||||
|
||||
void LlamaGenerator::ContextDeleter::operator()(
|
||||
llama_context* context) const noexcept {
|
||||
if (context != nullptr) {
|
||||
llama_free(context);
|
||||
}
|
||||
if (context != nullptr) {
|
||||
llama_free(context);
|
||||
}
|
||||
}
|
||||
|
||||
LlamaGenerator::LlamaGenerator(
|
||||
@@ -40,61 +40,61 @@ LlamaGenerator::LlamaGenerator(
|
||||
logger_(std::move(logger)),
|
||||
prompt_formatter_(std::move(prompt_formatter)),
|
||||
prompt_directory_(std::move(prompt_directory)) {
|
||||
if (model_path.empty()) {
|
||||
throw std::runtime_error("LlamaGenerator: model path must not be empty");
|
||||
}
|
||||
if (model_path.empty()) {
|
||||
throw std::runtime_error("LlamaGenerator: model path must not be empty");
|
||||
}
|
||||
|
||||
if (!prompt_formatter_) {
|
||||
throw std::runtime_error(
|
||||
"LlamaGenerator: prompt formatter dependency must not be null");
|
||||
}
|
||||
if (!prompt_formatter_) {
|
||||
throw std::runtime_error(
|
||||
"LlamaGenerator: prompt formatter dependency must not be null");
|
||||
}
|
||||
|
||||
if (!prompt_directory_) {
|
||||
throw std::runtime_error(
|
||||
"LlamaGenerator: prompt directory dependency must not be null");
|
||||
}
|
||||
if (!prompt_directory_) {
|
||||
throw std::runtime_error(
|
||||
"LlamaGenerator: prompt directory dependency must not be null");
|
||||
}
|
||||
|
||||
const auto sampling = options.generator.sampling.value_or(SamplingOptions{});
|
||||
const auto sampling = options.generator.sampling.value_or(SamplingOptions{});
|
||||
|
||||
if (sampling.temperature < 0.0F) {
|
||||
throw std::runtime_error(
|
||||
"LlamaGenerator: sampling temperature must be >= 0");
|
||||
}
|
||||
if (sampling.temperature < 0.0F) {
|
||||
throw std::runtime_error(
|
||||
"LlamaGenerator: sampling temperature must be >= 0");
|
||||
}
|
||||
|
||||
if (sampling.top_p <= 0.0F || sampling.top_p > 1.0F) {
|
||||
throw std::runtime_error(
|
||||
"LlamaGenerator: sampling top-p must be in (0, 1]");
|
||||
}
|
||||
if (sampling.top_p <= 0.0F || sampling.top_p > 1.0F) {
|
||||
throw std::runtime_error(
|
||||
"LlamaGenerator: sampling top-p must be in (0, 1]");
|
||||
}
|
||||
|
||||
if (sampling.top_k == 0U) {
|
||||
throw std::runtime_error("LlamaGenerator: sampling top-k must be > 0");
|
||||
}
|
||||
if (sampling.top_k == 0U) {
|
||||
throw std::runtime_error("LlamaGenerator: sampling top-k must be > 0");
|
||||
}
|
||||
|
||||
if (sampling.seed < -1) {
|
||||
throw std::runtime_error(
|
||||
"LlamaGenerator: seed must be >= 0, or -1 for random");
|
||||
}
|
||||
if (sampling.seed < -1) {
|
||||
throw std::runtime_error(
|
||||
"LlamaGenerator: seed must be >= 0, or -1 for random");
|
||||
}
|
||||
|
||||
if (sampling.n_ctx == 0 || sampling.n_ctx > kMaxContextSize) {
|
||||
throw std::runtime_error(
|
||||
"LlamaGenerator: context size must be in range [1, 32768]");
|
||||
}
|
||||
if (sampling.n_ctx == 0 || sampling.n_ctx > kMaxContextSize) {
|
||||
throw std::runtime_error(
|
||||
"LlamaGenerator: context size must be in range [1, 32768]");
|
||||
}
|
||||
|
||||
sampling_temperature_ = sampling.temperature;
|
||||
sampling_top_p_ = sampling.top_p;
|
||||
sampling_top_k_ = sampling.top_k;
|
||||
sampling_temperature_ = sampling.temperature;
|
||||
sampling_top_p_ = sampling.top_p;
|
||||
sampling_top_k_ = sampling.top_k;
|
||||
|
||||
if (sampling.seed == -1) {
|
||||
std::random_device random_device;
|
||||
rng_.seed(random_device());
|
||||
} else {
|
||||
rng_.seed(static_cast<uint32_t>(sampling.seed));
|
||||
}
|
||||
if (sampling.seed == -1) {
|
||||
std::random_device random_device;
|
||||
rng_.seed(random_device());
|
||||
} else {
|
||||
rng_.seed(static_cast<uint32_t>(sampling.seed));
|
||||
}
|
||||
|
||||
n_ctx_ = sampling.n_ctx;
|
||||
n_gpu_layers_ = sampling.n_gpu_layers;
|
||||
n_ctx_ = sampling.n_ctx;
|
||||
n_gpu_layers_ = sampling.n_gpu_layers;
|
||||
|
||||
this->Load(model_path);
|
||||
this->Load(model_path);
|
||||
}
|
||||
|
||||
LlamaGenerator::~LlamaGenerator() = default;
|
||||
|
||||
@@ -4,15 +4,15 @@
|
||||
* context, and resets prior resources during model initialization.
|
||||
*/
|
||||
|
||||
#include <ggml-backend.h>
|
||||
#include <llama.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
#include <ggml-backend.h>
|
||||
#include <llama.h>
|
||||
|
||||
#include "data_generation/llama_generator.h"
|
||||
|
||||
// Maximum batch size for decode operations. Capping the batch prevents
|
||||
@@ -20,42 +20,42 @@
|
||||
static constexpr uint32_t kMaxBatchSize = 5000U;
|
||||
|
||||
void LlamaGenerator::Load(const std::string& model_path) {
|
||||
context_.reset();
|
||||
model_.reset();
|
||||
context_.reset();
|
||||
model_.reset();
|
||||
|
||||
// Specifically load dynamic ggml backends (like CUDA) that are provided
|
||||
// externally before attempting to load a model.
|
||||
ggml_backend_load_all();
|
||||
// Specifically load dynamic ggml backends (like CUDA) that are provided
|
||||
// externally before attempting to load a model.
|
||||
ggml_backend_load_all();
|
||||
|
||||
llama_model_params model_params = llama_model_default_params();
|
||||
model_params.n_gpu_layers = n_gpu_layers_;
|
||||
llama_model_params model_params = llama_model_default_params();
|
||||
model_params.n_gpu_layers = n_gpu_layers_;
|
||||
|
||||
ModelHandle loaded_model(
|
||||
llama_model_load_from_file(model_path.c_str(), model_params));
|
||||
ModelHandle loaded_model(
|
||||
llama_model_load_from_file(model_path.c_str(), model_params));
|
||||
|
||||
if (!loaded_model) {
|
||||
throw std::runtime_error(
|
||||
"LlamaGenerator: failed to load model from path: " + model_path);
|
||||
}
|
||||
if (!loaded_model) {
|
||||
throw std::runtime_error(
|
||||
"LlamaGenerator: failed to load model from path: " + model_path);
|
||||
}
|
||||
|
||||
llama_context_params context_params = llama_context_default_params();
|
||||
context_params.n_ctx = n_ctx_;
|
||||
context_params.n_batch = std::min(n_ctx_, kMaxBatchSize);
|
||||
llama_context_params context_params = llama_context_default_params();
|
||||
context_params.n_ctx = n_ctx_;
|
||||
context_params.n_batch = std::min(n_ctx_, kMaxBatchSize);
|
||||
|
||||
ContextHandle loaded_context(
|
||||
llama_init_from_model(loaded_model.get(), context_params));
|
||||
ContextHandle loaded_context(
|
||||
llama_init_from_model(loaded_model.get(), context_params));
|
||||
|
||||
if (!loaded_context) {
|
||||
throw std::runtime_error("LlamaGenerator: failed to create context");
|
||||
}
|
||||
if (!loaded_context) {
|
||||
throw std::runtime_error("LlamaGenerator: failed to create context");
|
||||
}
|
||||
|
||||
model_ = std::move(loaded_model);
|
||||
context_ = std::move(loaded_context);
|
||||
model_ = std::move(loaded_model);
|
||||
context_ = std::move(loaded_context);
|
||||
|
||||
if (logger_) {
|
||||
logger_->Log({.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::Startup,
|
||||
.message = std::format("[LlamaGenerator] Loaded model: {} ",
|
||||
model_path)});
|
||||
}
|
||||
if (logger_) {
|
||||
logger_->Log({.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::Startup,
|
||||
.message = std::format("[LlamaGenerator] Loaded model: {} ",
|
||||
model_path)});
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user