AaronPo/the-biergarten-app
1
0
Fork 0
mirror of https://github.com/aaronpo97/the-biergarten-app.git synced 2026-10-14 17:54:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update generate pipeline to add OpenAI api as an additional backend (for comparison testing), add brewery coordinates (#244)

Browse Source 
* add brewery coordinates

* Add openai mode

* Add coordinates to schema

* Fix C++ review findings in pipeline: k-prefixed enum values, include ordering, populated validation error messages, conventional const notation, and endl replaced with newline
This commit is contained in:
Aaron Po
2026-08-23 15:49:28 -04:00
committed by Aaron Po
parent 31da1d557c
commit e738fe484a
35 changed files with 1327 additions and 343 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
tooling/pipeline/CMakeLists.txt
View File

@@ -192,6 +192,19 @@ target_sources(${PROJECT_NAME} PRIVATE
src/data_generation/mock/deterministic_hash.cc
)
# --- data_generation: shared JSON validation (always built — no llama.cpp dependency) ---
target_sources(${PROJECT_NAME} PRIVATE
src/data_generation/generation_json_validation.cc
)
# --- data_generation: openai (always built — no llama.cpp dependency) ---
target_sources(${PROJECT_NAME} PRIVATE
src/data_generation/openai/openai_generator.cc
src/data_generation/openai/call_chat_completions_api.cc
src/data_generation/openai/generate_brewery.cc
src/data_generation/openai/generate_user.cc
)
# --- data_generation: llama (skipped for mock-only builds) ---
if(NOT BIERGARTEN_MOCK_ONLY)
target_sources(${PROJECT_NAME} PRIVATE

3
tooling/pipeline/includes/biergarten_pipeline.h
View File

@@ -15,12 +15,15 @@
// --- data_generation ---
#include "data_generation/data_generator.h"
#include "data_generation/generation_json_validation.h"
#include "data_generation/json_grammars.h"
#ifndef BIERGARTEN_MOCK_ONLY
#include "data_generation/llama_generator.h"
#include "data_generation/llama_generator_helpers.h"
#endif
#include "data_generation/mock_generator.h"
#include "data_generation/openai_generator.h"
#include "data_generation/openai_json_schemas.h"
#include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h"
#include "data_generation/prompt_formatting/prompt_formatter.h"

52
tooling/pipeline/includes/data_generation/generation_json_validation.h Normal file
View File

@@ -0,0 +1,52 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_GENERATION_JSON_VALIDATION_H_
#define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_GENERATION_JSON_VALIDATION_H_
/**
* @file data_generation/generation_json_validation.h
* @brief Generator-agnostic helpers for preparing prompt context and
* validating structured JSON output. Shared across all DataGenerator
* implementations (Llama, OpenAI, ...).
*/
#include <cstddef>
#include <optional>
#include <string>
#include <string_view>
#include "data_model/generated_models.h"
/**
* @brief Normalizes and truncates regional context.
*
* @param region_context Input regional context text.
* @param max_chars Maximum output length.
* @return Processed region context.
*/
std::string PrepareRegionContext(std::string_view region_context,
size_t max_chars = 2000);
/**
* @brief Validates and parses brewery JSON output.
*
* @param raw Raw model output.
* @param brewery_out Parsed brewery payload.
* @return Validation error message if invalid, or std::nullopt on success.
*/
std::optional<std::string> ValidateBreweryJson(const std::string& raw,
BreweryResult& brewery_out);
/**
* @brief Validates and parses user JSON output.
*
* Only populates `username`, `bio`, and `activity_weight` -- `first_name`
* and `last_name` are not LLM-authored and are set separately from the
* sampled Name.
*
* @param raw Raw model output.
* @param user_out Parsed user payload.
* @return Validation error message if invalid, or std::nullopt on success.
*/
std::optional<std::string> ValidateUserJson(const std::string& raw,
UserResult& user_out);
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_GENERATION_JSON_VALIDATION_H_

43
tooling/pipeline/includes/data_generation/llama_generator_helpers.h
View File

@@ -3,30 +3,17 @@
/**
* @file data_generation/llama_generator_helpers.h
* @brief Shared helper APIs used by LlamaGenerator translation units.
* @brief Llama.cpp-specific helper APIs used by LlamaGenerator translation
* units. Generator-agnostic helpers (prompt-context prep, JSON validation)
* live in data_generation/generation_json_validation.h instead.
*/
#include <cstddef>
#include <cstdint>
#include <optional>
#include <string>
#include <string_view>
#include "data_model/generated_models.h"
struct llama_vocab;
using llama_token = int32_t;
/**
* @brief Normalizes and truncates regional context.
*
* @param region_context Input regional context text.
* @param max_chars Maximum output length.
* @return Processed region context.
*/
std::string PrepareRegionContext(std::string_view region_context,
size_t max_chars = 2000);
/**
* @brief Decodes a sampled token and appends it to output text.
*
@@ -37,28 +24,4 @@ std::string PrepareRegionContext(std::string_view region_context,
void AppendTokenPiece(const llama_vocab* vocab, llama_token token,
std::string& output);
/**
* @brief Validates and parses brewery JSON output.
*
* @param raw Raw model output.
* @param brewery_out Parsed brewery payload.
* @return Validation error message if invalid, or std::nullopt on success.
*/
std::optional<std::string> ValidateBreweryJson(const std::string& raw,
BreweryResult& brewery_out);
/**
* @brief Validates and parses user JSON output.
*
* Only populates `username`, `bio`, and `activity_weight` -- `first_name`
* and `last_name` are not LLM-authored and are set separately from the
* sampled Name.
*
* @param raw Raw model output.
* @param user_out Parsed user payload.
* @return Validation error message if invalid, or std::nullopt on success.
*/
std::optional<std::string> ValidateUserJson(const std::string& raw,
UserResult& user_out);
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_HELPERS_H_

104
tooling/pipeline/includes/data_generation/openai_generator.h Normal file
View File

@@ -0,0 +1,104 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_OPENAI_GENERATOR_H_
#define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_OPENAI_GENERATOR_H_
/**
* @file data_generation/openai_generator.h
* @brief OpenAI Chat Completions API-backed implementation of DataGenerator.
*/
#include <memory>
#include <string>
#include <string_view>
#include "data_generation/data_generator.h"
#include "data_model/models.h"
#include "services/logging/logger.h"
#include "services/prompting/prompt_directory.h"
#include "web_client/web_client.h"
/**
* @brief Data generator implementation backed by the OpenAI Chat
* Completions API.
*
* Unlike LlamaGenerator, this generator has no local model state and no
* chat-template formatting step -- Chat Completions takes system/user
* messages directly, and Structured Outputs (response_format: json_schema,
* strict: true) guarantee schema-valid JSON, replacing llama.cpp's GBNF
* grammar constraint.
*/
class OpenAIGenerator final : public DataGenerator {
public:
/**
* @brief Constructs a generator that calls the OpenAI Chat Completions
* API.
*
* @param api_key OpenAI API key (from the OPENAI_API_KEY environment
* variable -- never accepted as a CLI argument).
* @param model OpenAI model ID, e.g. "gpt-4o-mini".
* @param logger Sink for pipeline diagnostics.
* @param prompt_directory Directory service for loading named prompt
* files (the same BREWERY_GENERATION/USER_GENERATION prompts Llama
* uses -- they're backend-agnostic).
* @param web_client HTTP client used to reach the Chat Completions API.
*/
OpenAIGenerator(std::string api_key, std::string model,
std::shared_ptr<ILogger> logger,
std::unique_ptr<IPromptDirectory> prompt_directory,
std::unique_ptr<WebClient> web_client);
~OpenAIGenerator() override = default;
OpenAIGenerator(const OpenAIGenerator&) = delete;
OpenAIGenerator& operator=(const OpenAIGenerator&) = delete;
OpenAIGenerator(OpenAIGenerator&&) = delete;
OpenAIGenerator& operator=(OpenAIGenerator&&) = delete;
/**
* @brief Generates brewery data for a specific location.
*
* @param enriched_city Enriched city the brewery is associated with.
* @return Generated brewery result.
*/
BreweryResult GenerateBrewery(const EnrichedCity& enriched_city) override;
/**
* @brief Generates a user profile grounded in a sampled name and persona.
*
* @param city Enriched city the user is associated with.
* @param persona Persona archetype grounding the generated bio.
* @param name Sampled first/last name -- not LLM-invented.
* @return Generated user profile.
*/
UserResult GenerateUser(const EnrichedCity& city, const UserPersona& persona,
const Name& name) override;
private:
/**
* @brief Sends a single Chat Completions request with a Structured
* Outputs schema and returns the raw JSON text of the response.
*
* @param system_prompt System role message content.
* @param user_prompt User role message content.
* @param json_schema Raw JSON Schema text constraining the response
* shape (see data_generation/openai_json_schemas.h).
* @param schema_name Identifier for the schema, required by the
* response_format.json_schema wrapper.
* @param max_tokens Maximum completion tokens to generate.
* @return Raw JSON text produced by the model.
* @throws std::runtime_error on a request/HTTP failure, a safety-policy
* refusal, or a response with no message content.
*/
std::string CallChatCompletionsApi(const std::string& system_prompt,
const std::string& user_prompt,
std::string_view json_schema,
std::string_view schema_name,
int max_tokens);
std::string api_key_;
std::string model_;
std::shared_ptr<ILogger> logger_;
std::unique_ptr<IPromptDirectory> prompt_directory_;
std::unique_ptr<WebClient> web_client_;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_OPENAI_GENERATOR_H_

46
tooling/pipeline/includes/data_generation/openai_json_schemas.h Normal file
View File

@@ -0,0 +1,46 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_OPENAI_JSON_SCHEMAS_H_
#define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_OPENAI_JSON_SCHEMAS_H_
/**
* @file data_generation/openai_json_schemas.h
* @brief JSON Schemas constraining structured output from OpenAIGenerator
* Chat Completions requests, via response_format.json_schema (Structured
* Outputs, strict mode). This is OpenAI's equivalent of the GBNF grammars
* in json_grammars.h -- the API guarantees the response message content is
* a single JSON string that validates against the schema, so there's no
* grammar-level "thought-block" prefix to account for.
*
* OpenAI's strict mode requires "additionalProperties": false on every
* object in the schema, which these already have.
*/
#include <string_view>
inline constexpr std::string_view kBreweryJsonSchema = R"json_brewery(
{
"type": "object",
"properties": {
"name_en": {"type": "string"},
"description_en": {"type": "string"},
"name_local": {"type": "string"},
"description_local": {"type": "string"}
},
"required": ["name_en", "description_en", "name_local", "description_local"],
"additionalProperties": false
}
)json_brewery";
inline constexpr std::string_view kUserJsonSchema = R"json_user(
{
"type": "object",
"properties": {
"username": {"type": "string"},
"bio": {"type": "string"},
"activity_weight": {"type": "number"}
},
"required": ["username", "bio", "activity_weight"],
"additionalProperties": false
}
)json_user";
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_OPENAI_JSON_SCHEMAS_H_

2
tooling/pipeline/includes/data_model/generated_models.h
View File

@@ -94,6 +94,8 @@ struct EnrichedCity {
struct BreweryAddress {
City city;
std::string postal_code{};
double longitude{};
double latitude{};
};
/**

35
tooling/pipeline/includes/data_model/models.h
View File

@@ -66,6 +66,9 @@ struct City {
*/
std::string iso3166_1{};
double longitude{};
double latitude{};
/**
* @brief Local language codes in priority order.
*/
@@ -184,25 +187,47 @@ struct SamplingOptions {
int n_gpu_layers = 0;
};
/**
* @brief Selects which DataGenerator backend the pipeline uses.
*/
enum class GeneratorMode {
kMock,
kLlama,
kOpenAI,
};
/**
* @brief Configuration for the LLM generator component.
*/
struct GeneratorOptions {
/**
* @brief Path to the LLM model file (gguf format).
* @brief Path to the LLM model file (gguf format). Only meaningful for
* GeneratorMode::kLlama.
*/
std::filesystem::path model_path;
/**
* @brief Use mocked generator instead of actual LLM inference.
* @brief Which generator backend to use.
*/
bool use_mocked = false;
GeneratorMode mode = GeneratorMode::kMock;
/**
* @brief Specific sampling parameters for this generator.
* If nullopt, the application should use global defaults.
* @brief Specific sampling parameters for this generator. Only
* meaningful for GeneratorMode::kLlama; if nullopt, LlamaGenerator uses
* its own defaults.
*/
std::optional<SamplingOptions> sampling;
/**
* @brief OpenAI API key, read from the OPENAI_API_KEY environment
* variable. Only populated for GeneratorMode::kOpenAI.
*/
std::string openai_api_key;
/**
* @brief OpenAI model ID used by OpenAIGenerator.
*/
std::string openai_model = "gpt-4o-mini";
};
/**

18
tooling/pipeline/includes/services/database/sqlite_statement_helpers.h
View File

@@ -26,6 +26,8 @@ CREATE TABLE IF NOT EXISTS cities (
iso3166_2 TEXT NOT NULL,
country TEXT NOT NULL,
iso3166_1 TEXT NOT NULL,
longitude REAL NOT NULL,
latitude REAL NOT NULL,
local_languages_json TEXT NOT NULL,
postal_code_country_format_regex TEXT NOT NULL,
postal_code_city_regex_json TEXT NOT NULL,
@@ -53,6 +55,8 @@ CREATE TABLE IF NOT EXISTS brewery_addresses (
brewery_id INTEGER NOT NULL,
city_id INTEGER NOT NULL,
postal_code TEXT NOT NULL,
longitude REAL NOT NULL,
latitude REAL NOT NULL,
FOREIGN KEY(brewery_id) REFERENCES breweries(id) ON DELETE CASCADE,
FOREIGN KEY(city_id) REFERENCES cities(id) ON DELETE CASCADE
);
@@ -105,10 +109,12 @@ INSERT INTO cities (
iso3166_2,
country,
iso3166_1,
longitude,
latitude,
local_languages_json,
postal_code_country_format_regex,
postal_code_city_regex_json
) VALUES (?, ?, ?, ?, ?, ?, ?, ?);
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
)sql";
inline constexpr std::string_view kInsertBrewerySql = R"sql(
@@ -124,8 +130,10 @@ inline constexpr std::string_view kInsertBreweryAddressSql = R"sql(
INSERT INTO brewery_addresses (
brewery_id,
city_id,
postal_code
) VALUES (?, ?, ?);
postal_code,
longitude,
latitude
) VALUES (?, ?, ?, ?, ?);
)sql";
inline constexpr std::string_view kInsertUserSql = R"sql(
@@ -157,6 +165,8 @@ enum CityBindIndex {
kCityIso31662BindIndex,
kCityCountryBindIndex,
kCityIso31661BindIndex,
kCityLongitudeBindIndex,
kCityLatitudeBindIndex,
kCityLanguagesBindIndex,
kCityPostalCodeCountryFormatRegexBindIndex,
kCityPostalCodeCityRegexJsonBindIndex,
@@ -173,6 +183,8 @@ enum BreweryAddressBindIndex {
kBreweryAddressBreweryIdBindIndex = 1,
kBreweryAddressCityIdBindIndex,
kBreweryAddressPostalCodeBindIndex,
kBreweryAddressLongitudeBindIndex,
kBreweryAddressLatitudeBindIndex,
};
enum UserBindIndex {

17
tooling/pipeline/includes/web_client/http_web_client.h
View File

@@ -9,6 +9,7 @@
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "services/logging/logger.h"
#include "web_client/web_client.h"
@@ -39,6 +40,22 @@ class HttpWebClient final : public WebClient {
*/
std::string Get(const std::string& url) override;
/**
* @brief Executes a blocking HTTP/HTTPS POST request against a full URL.
*
* Sends @p body with a "Content-Type: application/json" header, plus any
* additional headers supplied by the caller (e.g. authentication).
*
* @param url Fully-qualified URL.
* @param body Request body, sent as-is.
* @param headers Additional request headers.
* @return Response body on HTTP 2xx; throws std::runtime_error otherwise.
*/
std::string Post(
const std::string& url, const std::string& body,
const std::vector<std::pair<std::string, std::string>>& headers)
override;
/**
* @brief Percent-encodes a single URI component (query parameter value or
* path segment). Delegates to httplib::encode_uri_component().

16
tooling/pipeline/includes/web_client/web_client.h
View File

@@ -7,6 +7,8 @@
*/
#include <string>
#include <utility>
#include <vector>
/**
* @brief Abstract web client interface.
@@ -23,6 +25,20 @@ class WebClient {
*/
virtual std::string Get(const std::string& url) = 0;
/**
* @brief Executes an HTTP POST request with a JSON body.
*
* @param url Request URL.
* @param body Request body (sent as-is; callers are responsible for
* providing valid JSON when the target API expects it).
* @param headers Additional request headers (e.g. authentication),
* sent alongside a fixed "Content-Type: application/json" header.
* @return Response body.
*/
virtual std::string Post(
const std::string& url, const std::string& body,
const std::vector<std::pair<std::string, std::string>>& headers) = 0;
/**
* @brief URL-encodes a string value.
*

64
tooling/pipeline/src/application_options/parse_arguments.cc
View File

@@ -1,4 +1,5 @@
#include <chrono>
#include <cstdlib>
#include <format>
#include <iostream>
#include <optional>
@@ -37,13 +38,20 @@ std::optional<ApplicationOptions> ParseArguments(
"Number of layers to offload to GPU");
};
// --mocked and --model are mutually exclusive; validation is enforced below
// rather than at registration to produce a clear diagnostic message.
// --mocked, --model, and --openai are mutually exclusive; validation is
// enforced below rather than at registration to produce a clear
// diagnostic message.
auto add_generator_options = [&]() -> void {
opt("mocked", prog_opts::bool_switch(),
"Use mocked generator for brewery/user data");
opt("model,m", prog_opts::value<std::string>()->default_value(""),
"Path to LLM model (gguf)");
opt("openai", prog_opts::bool_switch(),
"Use the OpenAI API (Chat Completions) for brewery/user data. "
"Requires the OPENAI_API_KEY environment variable.");
opt("openai-model",
prog_opts::value<std::string>()->default_value("gpt-4o-mini"),
"OpenAI model ID used when --openai is set");
};
auto add_pipeline_options = [&]() -> void {
@@ -109,14 +117,19 @@ std::optional<ApplicationOptions> ParseArguments(
const bool use_mocked = var_map["mocked"].as<bool>();
const std::string model_path = var_map["model"].as<std::string>();
const bool use_openai = var_map["openai"].as<bool>();
const int n_gpu_layers = var_map["n-gpu-layers"].as<int>();
// Enforce mutual exclusivity before any further configuration is
// applied.
if (use_mocked && !model_path.empty()) {
const int selected_generator_count =
(use_mocked ? 1 : 0) + (!model_path.empty() ? 1 : 0) +
(use_openai ? 1 : 0);
if (selected_generator_count > 1) {
const std::string msg =
"Invalid arguments: --mocked and --model are mutually "
"exclusive";
"Invalid arguments: --mocked, --model, and --openai are "
"mutually exclusive";
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
@@ -127,10 +140,10 @@ std::optional<ApplicationOptions> ParseArguments(
return std::nullopt;
}
if (!use_mocked && model_path.empty()) {
if (selected_generator_count == 0) {
const std::string msg =
"Invalid arguments: either --mocked or --model must be "
"specified";
"Invalid arguments: exactly one of --mocked, --model, or "
"--openai must be specified";
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
@@ -141,8 +154,9 @@ std::optional<ApplicationOptions> ParseArguments(
return std::nullopt;
}
// Prompt directory is only meaningful for live inference — the mock
// generator has no use for it and should not require it to be present.
// Prompt directory is only meaningful for live inference (Llama or
// OpenAI) — the mock generator has no use for it and should not
// require it to be present.
if (!use_mocked && options.pipeline.prompt_dir.empty()) {
const std::string msg =
"Invalid arguments: --prompt-dir is required when not using "
@@ -157,8 +171,29 @@ std::optional<ApplicationOptions> ParseArguments(
return std::nullopt;
}
options.generator.use_mocked = use_mocked;
if (use_openai) {
const char* api_key_env = std::getenv("OPENAI_API_KEY");
if (api_key_env == nullptr || api_key_env[0] == '\0') {
const std::string msg =
"Invalid arguments: the OPENAI_API_KEY environment "
"variable must be set when using --openai";
if (logger) {
logger->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = msg});
} else {
std::cerr << msg << '\n';
}
return std::nullopt;
}
options.generator.openai_api_key = api_key_env;
}
options.generator.mode = use_mocked ? GeneratorMode::kMock
: use_openai ? GeneratorMode::kOpenAI
: GeneratorMode::kLlama;
options.generator.model_path = model_path;
options.generator.openai_model = var_map["openai-model"].as<std::string>();
// Only populate sampling config when the user explicitly overrides at
// least one value. Leaving it as std::nullopt lets LlamaGenerator fall
@@ -172,10 +207,11 @@ std::optional<ApplicationOptions> ParseArguments(
if (user_provided_sampling) {
// Warn but do not fail — the run is still valid, the flags are just
// silently irrelevant when no model is loaded.
if (use_mocked) {
// silently irrelevant when no local model is loaded.
if (options.generator.mode != GeneratorMode::kLlama) {
const std::string msg =
"Sampling parameters are ignored when using --mocked";
"Sampling parameters are ignored unless using --model "
"(Llama)";
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Warn,
.phase = PipelinePhase::Startup,

59
tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_breweries.cc
View File

@@ -4,12 +4,55 @@
*/
#include <chrono>
#include <cmath>
#include <format>
#include <numbers>
#include <optional>
#include <random>
#include "biergarten_pipeline_orchestrator.h"
#include "services/logging/logger.h"
struct Coords {
double longitude;
double latitude;
};
// @todo find a better home for this
namespace {
constexpr double kEarthRadiusKm = 6371.0;
double DegreesToRadians(double degrees) {
return degrees * std::numbers::pi / 180.0;
}
double RadiansToDegrees(double radians) {
return radians * 180.0 / std::numbers::pi;
}
} // namespace
static Coords GetRandomCoordsWithinRange(const Coords& centre,
const int distance_km,
std::mt19937& rng) {
std::uniform_real_distribution<double> angle_dist(0.0,
2.0 * std::numbers::pi);
std::uniform_real_distribution<double> unit_dist(0.0, 1.0);
const double angle_radians = angle_dist(rng);
// sqrt() keeps points uniformly distributed over the disc's area rather
// than clustering near the centre.
const double radius_km = distance_km * std::sqrt(unit_dist(rng));
const double delta_latitude_degrees =
RadiansToDegrees(radius_km * std::cos(angle_radians) / kEarthRadiusKm);
const double delta_longitude_degrees = RadiansToDegrees(
radius_km * std::sin(angle_radians) /
(kEarthRadiusKm * std::cos(DegreesToRadians(centre.latitude))));
return Coords{.longitude = centre.longitude + delta_longitude_degrees,
.latitude = centre.latitude + delta_latitude_degrees};
}
void BiergartenPipelineOrchestrator::GenerateBreweries(
std::span<const EnrichedCity> cities) {
logger_->Log({.level = LogLevel::Info,
@@ -19,18 +62,28 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
generated_breweries_.clear();
size_t skipped_count = 0;
size_t export_failed_count = 0;
std::mt19937 rng(std::random_device{}());
const auto generate_record =
[this, &skipped_count](
[this, &skipped_count, &rng](
const EnrichedCity& enriched_city) -> std::optional<BreweryRecord> {
try {
const BreweryResult brewery =
generator_->GenerateBrewery(enriched_city);
const std::string postal_code =
postal_code_service_->GeneratePostalCode(enriched_city.location);
constexpr int kMaxDistanceFromCentreKm = 5;
const Coords city_centre{.longitude = enriched_city.location.longitude,
.latitude = enriched_city.location.latitude};
const Coords brewery_coords = GetRandomCoordsWithinRange(
city_centre, kMaxDistanceFromCentreKm, rng);
return BreweryRecord{
.address = BreweryAddress{.city = enriched_city.location,
.postal_code = postal_code},
.postal_code = postal_code,
.longitude = brewery_coords.longitude,
.latitude = brewery_coords.latitude},
.brewery = brewery};
} catch (const std::exception& e) {
++skipped_count;
@@ -91,4 +144,4 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
"brewery/breweries to SQLite",
export_failed_count)});
}
}
}

42
tooling/pipeline/src/biergarten_pipeline_orchestrator/log_results.cc
View File

@@ -17,17 +17,19 @@ void BiergartenPipelineOrchestrator::LogResults() const {
for (const auto& [address, brewery] : generated_breweries_) {
const City& city = address.city;
brewery_output.push_back(boost::json::object{
{"name_en", brewery.name_en},
{"description_en", brewery.description_en},
{"name_local", brewery.name_local},
{"description_local", brewery.description_local},
{"address", boost::json::object{
{"city", city.city},
{"country", city.country},
{"state_province", city.state_province},
{"iso3166_2", city.iso3166_2},
{"postal_code", address.postal_code},
}}});
{"name_en", brewery.name_en},
{"description_en", brewery.description_en},
{"name_local", brewery.name_local},
{"description_local", brewery.description_local},
{"address", boost::json::object{
{"city", city.city},
{"country", city.country},
{"state_province", city.state_province},
{"iso3166_2", city.iso3166_2},
{"postal_code", address.postal_code},
{"longitude", address.longitude},
{"latitude", address.latitude},
}}});
}
std::ostringstream brewery_oss;
@@ -40,14 +42,14 @@ void BiergartenPipelineOrchestrator::LogResults() const {
for (const auto& generated_user : generated_users_) {
user_output.push_back(boost::json::object{
{"first_name", generated_user.user.first_name},
{"last_name", generated_user.user.last_name},
{"gender", generated_user.user.gender},
{"username", generated_user.user.username},
{"bio", generated_user.user.bio},
{"activity_weight", generated_user.user.activity_weight},
{"email", generated_user.email},
{"date_of_birth", generated_user.date_of_birth},
{"first_name", generated_user.user.first_name},
{"last_name", generated_user.user.last_name},
{"gender", generated_user.user.gender},
{"username", generated_user.user.username},
{"bio", generated_user.user.bio},
{"activity_weight", generated_user.user.activity_weight},
{"email", generated_user.email},
{"date_of_birth", generated_user.date_of_birth},
});
}
@@ -56,4 +58,4 @@ void BiergartenPipelineOrchestrator::LogResults() const {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Teardown,
.message = user_oss.str()});
}
}

13
tooling/pipeline/src/biergarten_pipeline_orchestrator/query_cities_with_countries.cc
View File

@@ -39,11 +39,10 @@ std::vector<City> BiergartenPipelineOrchestrator::QueryLocations() {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(" Locations available: {}",
all_locations.size())});
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(" Sampled locations: {}",
sampled_locations.size())});
.message = std::format(
" Locations available: {}\n Sampled Locations: {}",
all_locations.size(), sampled_locations.size())});
return sampled_locations;
}
}

236
tooling/pipeline/src/data_generation/generation_json_validation.cc Normal file
View File

@@ -0,0 +1,236 @@
/**
* @file data_generation/generation_json_validation.cc
* @brief Generator-agnostic prompt-context preparation and structured JSON
* validation, shared across all DataGenerator implementations.
*/
#include "data_generation/generation_json_validation.h"
#include <algorithm>
#include <array>
#include <cctype>
#include <format>
#include <span>
#include <string>
#include <string_view>
#include <boost/json.hpp>
namespace {
/**
* String trimming: removes leading and trailing whitespace
*/
std::string Trim(std::string_view value) {
constexpr std::string_view whitespace = " \t\n\r\f\v";
const size_t first_index = value.find_first_not_of(whitespace);
if (first_index == std::string_view::npos) {
return {};
}
const size_t last_index = value.find_last_not_of(whitespace);
return std::string(value.substr(first_index, last_index - first_index + 1));
}
/**
* Normalize whitespace: collapses multiple spaces/tabs/newlines into single
* spaces
*/
std::string CondenseWhitespace(std::string_view text) {
std::string out;
out.reserve(text.size());
bool pending_space = false;
for (const char chr : text) {
if (std::isspace(static_cast<unsigned char>(chr)) != 0) {
if (!out.empty()) {
pending_space = true;
}
continue;
}
if (pending_space) {
out.push_back(' ');
pending_space = false;
}
out.push_back(chr);
}
return out;
}
// Guard against truncating in the first half of the string.
// This preserves the critical opening content and avoids cutting critical
// context words early in the region description.
constexpr size_t kTruncationGuardDivisor = 2;
bool ReadRequiredTrimmedStringField(const boost::json::object& obj,
std::string_view key, std::string& out,
std::string* error_out) {
const boost::json::value* field = obj.if_contains(key);
if (field == nullptr || !field->is_string()) {
if (error_out != nullptr) {
*error_out =
std::format("Missing or invalid string field: {}", key);
}
return false;
}
const auto& string_value = field->as_string();
out = Trim(std::string_view(string_value.data(), string_value.size()));
if (out.empty() && error_out != nullptr) {
*error_out = std::format("Field must not be empty: {}", key);
}
return !out.empty();
}
bool HasSchemaPlaceholder(std::span<std::string* const> values) {
for (const std::string* value : values) {
std::string lowered = *value;
std::ranges::transform(
lowered, lowered.begin(), [](const unsigned char character) {
return static_cast<char>(std::tolower(character));
});
if (lowered == "string") {
return true;
}
}
return false;
}
} // namespace
/**
* Truncate region context to fit within max length while preserving word
* boundaries
*/
std::string PrepareRegionContext(std::string_view region_context,
const size_t max_chars) {
std::string normalized = CondenseWhitespace(region_context);
if (normalized.size() <= max_chars) {
return normalized;
}
normalized.resize(max_chars);
const size_t last_space = normalized.find_last_of(' ');
if (last_space != std::string::npos &&
last_space > max_chars / kTruncationGuardDivisor) {
normalized.resize(last_space);
}
normalized += "...";
return normalized;
}
std::optional<std::string> ValidateBreweryJson(const std::string& raw,
BreweryResult& brewery_out) {
boost::system::error_code error_code;
const std::string_view raw_view(raw);
const size_t opening_brace = raw_view.find('{');
if (opening_brace == std::string_view::npos) {
return "JSON parse error: missing opening brace '{'";
}
const std::string_view json_payload = raw_view.substr(opening_brace);
boost::json::value json_value = boost::json::parse(json_payload, error_code);
if (error_code) {
return "JSON parse error: " + error_code.message();
}
if (!json_value.is_object()) {
return "JSON root must be an object";
}
const auto& obj = json_value.get_object();
if (obj.size() != 4) {
return "JSON object must contain exactly four keys";
}
std::string validation_error;
if (!ReadRequiredTrimmedStringField(obj, "name_en", brewery_out.name_en,
&validation_error)) {
return validation_error;
}
if (!ReadRequiredTrimmedStringField(obj, "description_en",
brewery_out.description_en,
&validation_error)) {
return validation_error;
}
if (!ReadRequiredTrimmedStringField(
obj, "name_local", brewery_out.name_local, &validation_error)) {
return validation_error;
}
if (!ReadRequiredTrimmedStringField(obj, "description_local",
brewery_out.description_local,
&validation_error)) {
return validation_error;
}
const std::array schema_placeholders = {
&brewery_out.name_en, &brewery_out.description_en,
&brewery_out.name_local, &brewery_out.description_local};
if (HasSchemaPlaceholder(schema_placeholders)) {
return "JSON appears to be a schema placeholder, not content";
}
return std::nullopt;
}
std::optional<std::string> ValidateUserJson(const std::string& raw,
UserResult& user_out) {
boost::system::error_code error_code;
const std::string_view raw_view(raw);
const size_t opening_brace = raw_view.find('{');
if (opening_brace == std::string_view::npos) {
return "JSON parse error: missing opening brace '{'";
}
const std::string_view json_payload = raw_view.substr(opening_brace);
boost::json::value json_value = boost::json::parse(json_payload, error_code);
if (error_code) {
return "JSON parse error: " + error_code.message();
}
if (!json_value.is_object()) {
return "JSON root must be an object";
}
const auto& obj = json_value.get_object();
if (obj.size() != 3) {
return "JSON object must contain exactly three keys";
}
std::string validation_error;
if (!ReadRequiredTrimmedStringField(obj, "username", user_out.username,
&validation_error)) {
return validation_error;
}
if (!ReadRequiredTrimmedStringField(obj, "bio", user_out.bio,
&validation_error)) {
return validation_error;
}
const boost::json::value* activity_weight_field =
obj.if_contains("activity_weight");
if (activity_weight_field == nullptr ||
!activity_weight_field->is_number()) {
return "Missing or invalid numeric field: activity_weight";
}
const double activity_weight = activity_weight_field->to_number<double>();
if (activity_weight < 0.0 || activity_weight > 1.0) {
return "activity_weight must be between 0 and 1";
}
user_out.activity_weight = static_cast<float>(activity_weight);
const std::array schema_placeholders = {&user_out.username, &user_out.bio};
if (HasSchemaPlaceholder(schema_placeholders)) {
return "JSON appears to be a schema placeholder, not content";
}
return std::nullopt;
}

2
tooling/pipeline/src/data_generation/llama/generate_brewery.cc
View File

@@ -12,9 +12,9 @@
#include <string_view>
#include <vector>
#include "data_generation/generation_json_validation.h"
#include "data_generation/json_grammars.h"
#include "data_generation/llama_generator.h"
#include "data_generation/llama_generator_helpers.h"
static std::string FormatLocalLanguageCodes(
const std::vector<std::string>& codes) {

2
tooling/pipeline/src/data_generation/llama/generate_user.cc
View File

@@ -10,9 +10,9 @@
#include <string>
#include <string_view>
#include "data_generation/generation_json_validation.h"
#include "data_generation/json_grammars.h"
#include "data_generation/llama_generator.h"
#include "data_generation/llama_generator_helpers.h"
static constexpr int kUserInitialMaxTokens = 1200;

221
tooling/pipeline/src/data_generation/llama/helpers.cc
View File

@@ -1,123 +1,17 @@
/**
* @file data_generation/llama/helpers.cc
* @brief Provides prompt formatting, whitespace normalization, response
* parsing, token decoding, and JSON validation helpers for Llama modules.
* @brief Token decoding helper for LlamaGenerator modules.
*/
#include <llama.h>
#include <algorithm>
#include <array>
#include <boost/json.hpp>
#include <cctype>
#include <optional>
#include <span>
#include <stdexcept>
#include <string>
#include <string_view>
#include <vector>
#include "data_generation/llama_generator_helpers.h"
namespace {
/**
* String trimming: removes leading and trailing whitespace
*/
std::string Trim(std::string_view value) {
constexpr std::string_view whitespace = " \t\n\r\f\v";
const size_t first_index = value.find_first_not_of(whitespace);
if (first_index == std::string_view::npos) {
return {};
}
const size_t last_index = value.find_last_not_of(whitespace);
return std::string(value.substr(first_index, last_index - first_index + 1));
}
/**
* Normalize whitespace: collapses multiple spaces/tabs/newlines into single
* spaces
*/
std::string CondenseWhitespace(std::string_view text) {
std::string out;
out.reserve(text.size());
bool pending_space = false;
for (const char chr : text) {
if (std::isspace(static_cast<unsigned char>(chr)) != 0) {
if (!out.empty()) {
pending_space = true;
}
continue;
}
if (pending_space) {
out.push_back(' ');
pending_space = false;
}
out.push_back(chr);
}
return out;
}
// Guard against truncating in the first half of the string.
// This preserves the critical opening content and avoids cutting critical
// context words early in the region description.
constexpr size_t kTruncationGuardDivisor = 2;
bool ReadRequiredTrimmedStringField(const boost::json::object& obj,
std::string_view key, std::string& out,
std::string* error_out) {
const boost::json::value* field = obj.if_contains(key);
if (field == nullptr || !field->is_string()) {
return false;
}
const auto& string_value = field->as_string();
out = Trim(std::string_view(string_value.data(), string_value.size()));
return !out.empty();
}
bool HasSchemaPlaceholder(std::span<std::string* const> values) {
for (const std::string* value : values) {
std::string lowered = *value;
std::ranges::transform(
lowered, lowered.begin(), [](const unsigned char character) {
return static_cast<char>(std::tolower(character));
});
if (lowered == "string") {
return true;
}
}
return false;
}
} // namespace
/**
* Truncate region context to fit within max length while preserving word
* boundaries
*/
std::string PrepareRegionContext(std::string_view region_context,
const size_t max_chars) {
std::string normalized = CondenseWhitespace(region_context);
if (normalized.size() <= max_chars) {
return normalized;
}
normalized.resize(max_chars);
const size_t last_space = normalized.find_last_of(' ');
if (last_space != std::string::npos &&
last_space > max_chars / kTruncationGuardDivisor) {
normalized.resize(last_space);
}
normalized += "...";
return normalized;
}
void AppendTokenPiece(const llama_vocab* vocab, llama_token token,
std::string& output) {
constexpr size_t initial_buffer_size = 256;
@@ -153,116 +47,3 @@ void AppendTokenPiece(const llama_vocab* vocab, llama_token token,
throw std::runtime_error(
"LlamaGenerator: failed to decode sampled token piece");
}
std::optional<std::string> ValidateBreweryJson(const std::string& raw,
BreweryResult& brewery_out) {
boost::system::error_code error_code;
const std::string_view raw_view(raw);
const size_t opening_brace = raw_view.find('{');
if (opening_brace == std::string_view::npos) {
return "JSON parse error: missing opening brace '{'";
}
const std::string_view json_payload = raw_view.substr(opening_brace);
boost::json::value json_value = boost::json::parse(json_payload, error_code);
if (error_code) {
return "JSON parse error: " + error_code.message();
}
if (!json_value.is_object()) {
return "JSON root must be an object";
}
const auto& obj = json_value.get_object();
if (obj.size() != 4) {
return "JSON object must contain exactly four keys";
}
std::string validation_error;
if (!ReadRequiredTrimmedStringField(obj, "name_en", brewery_out.name_en,
&validation_error)) {
return validation_error;
}
if (!ReadRequiredTrimmedStringField(obj, "description_en",
brewery_out.description_en,
&validation_error)) {
return validation_error;
}
if (!ReadRequiredTrimmedStringField(
obj, "name_local", brewery_out.name_local, &validation_error)) {
return validation_error;
}
if (!ReadRequiredTrimmedStringField(obj, "description_local",
brewery_out.description_local,
&validation_error)) {
return validation_error;
}
const std::array schema_placeholders = {
&brewery_out.name_en, &brewery_out.description_en,
&brewery_out.name_local, &brewery_out.description_local};
if (HasSchemaPlaceholder(schema_placeholders)) {
return "JSON appears to be a schema placeholder, not content";
}
return std::nullopt;
}
std::optional<std::string> ValidateUserJson(const std::string& raw,
UserResult& user_out) {
boost::system::error_code error_code;
const std::string_view raw_view(raw);
const size_t opening_brace = raw_view.find('{');
if (opening_brace == std::string_view::npos) {
return "JSON parse error: missing opening brace '{'";
}
const std::string_view json_payload = raw_view.substr(opening_brace);
boost::json::value json_value = boost::json::parse(json_payload, error_code);
if (error_code) {
return "JSON parse error: " + error_code.message();
}
if (!json_value.is_object()) {
return "JSON root must be an object";
}
const auto& obj = json_value.get_object();
if (obj.size() != 3) {
return "JSON object must contain exactly three keys";
}
std::string validation_error;
if (!ReadRequiredTrimmedStringField(obj, "username", user_out.username,
&validation_error)) {
return validation_error;
}
if (!ReadRequiredTrimmedStringField(obj, "bio", user_out.bio,
&validation_error)) {
return validation_error;
}
const boost::json::value* activity_weight_field =
obj.if_contains("activity_weight");
if (activity_weight_field == nullptr ||
!activity_weight_field->is_number()) {
return "Missing or invalid numeric field: activity_weight";
}
const double activity_weight = activity_weight_field->to_number<double>();
if (activity_weight < 0.0 || activity_weight > 1.0) {
return "activity_weight must be between 0 and 1";
}
user_out.activity_weight = static_cast<float>(activity_weight);
const std::array schema_placeholders = {&user_out.username, &user_out.bio};
if (HasSchemaPlaceholder(schema_placeholders)) {
return "JSON appears to be a schema placeholder, not content";
}
return std::nullopt;
}

182
tooling/pipeline/src/data_generation/openai/call_chat_completions_api.cc Normal file
View File

@@ -0,0 +1,182 @@
/**
* @file data_generation/openai/call_chat_completions_api.cc
* @brief OpenAIGenerator::CallChatCompletionsApi() -- builds and sends a
* single OpenAI Chat Completions request with a Structured Outputs JSON
* schema, and extracts the resulting message content.
*
* API docs last accessed 2026-08-23.
* Docs: https://platform.openai.com/docs/api-reference/chat/create
* Structured Outputs: https://platform.openai.com/docs/guides/structured-outputs
*/
#include <format>
#include <stdexcept>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
#include <boost/json.hpp>
#include "data_generation/openai_generator.h"
namespace {
// https://platform.openai.com/docs/api-reference/chat/create
constexpr std::string_view kChatCompletionsUrl =
"https://api.openai.com/v1/chat/completions";
} // namespace
/**
* @brief Sends a single Chat Completions request constrained by a JSON
* Schema and returns the assistant message's text content.
*
* Request/response fields follow the Chat Completions API reference:
* https://platform.openai.com/docs/api-reference/chat/create
* Structured Outputs (response_format, strict mode, refusal field):
* https://platform.openai.com/docs/guides/structured-outputs
*
* @param system_prompt Text sent as the "system" role message.
* @param user_prompt Text sent as the "user" role message.
* @param json_schema JSON Schema the response must conform to.
* @param schema_name Identifier for response_format.json_schema.name.
* @param max_tokens Value sent as "max_completion_tokens".
* @return The raw JSON string from choices[0].message.content.
* @throws std::runtime_error on invalid schema, API error, malformed
* response, model refusal, or missing content.
*/
std::string OpenAIGenerator::CallChatCompletionsApi(
const std::string& system_prompt, const std::string& user_prompt,
std::string_view json_schema, std::string_view schema_name,
int max_tokens) {
boost::system::error_code schema_error;
boost::json::value schema_value =
boost::json::parse(json_schema, schema_error);
bool schema_is_invalid = static_cast<bool>(schema_error);
if (schema_is_invalid) {
throw std::runtime_error(
"OpenAIGenerator: invalid embedded JSON schema: " +
schema_error.message());
}
// response_format: https://platform.openai.com/docs/guides/structured-outputs
boost::json::object json_schema_wrapper;
json_schema_wrapper["name"] = std::string(schema_name);
json_schema_wrapper["strict"] = true;
json_schema_wrapper["schema"] = std::move(schema_value);
boost::json::object response_format;
response_format["type"] = "json_schema";
response_format["json_schema"] = std::move(json_schema_wrapper);
boost::json::object system_message;
system_message["role"] = "system";
system_message["content"] = system_prompt;
boost::json::object user_message;
user_message["role"] = "user";
user_message["content"] = user_prompt;
boost::json::array messages;
messages.push_back(std::move(system_message));
messages.push_back(std::move(user_message));
boost::json::object request;
request["model"] = model_;
request["max_completion_tokens"] = max_tokens;
request["messages"] = std::move(messages);
request["response_format"] = std::move(response_format);
const std::string body = boost::json::serialize(request);
const std::vector<std::pair<std::string, std::string>> headers = {
{"Authorization", std::format("Bearer {}", api_key_)},
};
const std::string response_body =
web_client_->Post(std::string(kChatCompletionsUrl), body, headers);
boost::system::error_code response_error;
boost::json::value response_value =
boost::json::parse(response_body, response_error);
bool response_body_is_invalid = static_cast<bool>(response_error);
if (response_body_is_invalid) {
throw std::runtime_error(
"OpenAIGenerator: failed to parse API response: " +
response_error.message());
}
bool response_root_is_not_object = !response_value.is_object();
if (response_root_is_not_object) {
throw std::runtime_error(
"OpenAIGenerator: API response root is not an object");
}
const auto& response_obj = response_value.get_object();
// Error object: https://platform.openai.com/docs/api-reference/chat/create
const boost::json::value* error_field = response_obj.if_contains("error");
bool top_level_error_occurred =
error_field != nullptr && error_field->is_object();
if (top_level_error_occurred) {
std::string error_message = "unknown error";
if (const boost::json::value* message_field =
error_field->get_object().if_contains("message");
message_field != nullptr && message_field->is_string()) {
error_message = std::string(message_field->as_string());
}
throw std::runtime_error(
std::format("OpenAIGenerator: API request failed: {}", error_message));
}
const boost::json::value* choices_field = response_obj.
if_contains("choices");
bool choices_field_missing_or_invalid =
choices_field == nullptr || !choices_field->is_array() ||
choices_field->get_array().empty();
if (choices_field_missing_or_invalid) {
throw std::runtime_error(
"OpenAIGenerator: API response missing choices array");
}
const boost::json::value& first_choice = choices_field->get_array().front();
bool first_choice_is_not_object = !first_choice.is_object();
if (first_choice_is_not_object) {
throw std::runtime_error(
"OpenAIGenerator: API response choice is not an object");
}
const auto& choice_obj = first_choice.get_object();
const boost::json::value* message_field = choice_obj.if_contains("message");
bool message_field_missing_or_invalid =
message_field == nullptr || !message_field->is_object();
if (message_field_missing_or_invalid) {
throw std::runtime_error(
"OpenAIGenerator: API response choice missing message");
}
const auto& message_obj = message_field->get_object();
// Refusal field: https://platform.openai.com/docs/guides/structured-outputs
const boost::json::value* refusal_field = message_obj.if_contains("refusal");
bool model_refused_request =
refusal_field != nullptr && refusal_field->is_string() &&
!refusal_field->as_string().empty();
if (model_refused_request) {
throw std::runtime_error(std::format(
"OpenAIGenerator: request refused by safety policies: {}",
std::string(refusal_field->as_string())));
}
const boost::json::value* content_field = message_obj.if_contains("content");
bool content_field_missing_or_invalid =
content_field == nullptr || !content_field->is_string();
if (content_field_missing_or_invalid) {
throw std::runtime_error(
"OpenAIGenerator: API response message had no text content");
}
return std::string(content_field->as_string());
}

132
tooling/pipeline/src/data_generation/openai/generate_brewery.cc Normal file
View File

@@ -0,0 +1,132 @@
/**
* @file data_generation/openai/generate_brewery.cc
* @brief Builds brewery prompts with regional context, calls the OpenAI
* Chat Completions API, and validates structured JSON output for brewery
* records.
*/
#include <format>
#include <optional>
#include <stdexcept>
#include <string>
#include <vector>
#include "data_generation/generation_json_validation.h"
#include "data_generation/openai_generator.h"
#include "data_generation/openai_json_schemas.h"
namespace {
std::string FormatLocalLanguageCodes(
const std::vector<std::string>& codes) {
if (codes.empty()) {
return "Not provided";
}
std::string formatted;
for (const std::string& code : codes) {
if (!formatted.empty()) {
formatted += ", ";
}
formatted += code;
}
return formatted;
}
constexpr int kBreweryMaxTokens = 2800;
// Structured outputs already guarantee schema-valid JSON, so retries here
// are only for transient failures or the model returning placeholder text
// -- not for malformed JSON shape, unlike LlamaGenerator's grammar-assisted
// retry loop.
constexpr int kMaxAttempts = 2;
} // namespace
BreweryResult OpenAIGenerator::GenerateBrewery(
const EnrichedCity& enriched_city) {
const City& location = enriched_city.location;
const std::string& region_context = enriched_city.region_context;
const std::string safe_region_context = PrepareRegionContext(region_context);
const std::string local_language_codes =
FormatLocalLanguageCodes(location.local_languages);
/**
* Load brewery system prompt via the injected prompt directory. The
* same file LlamaGenerator uses -- the prompt content is backend
* agnostic.
*/
const std::string system_prompt =
prompt_directory_->Load("BREWERY_GENERATION");
std::string user_prompt = std::format(
"## CITY:\n{}\n\n## COUNTRY:\n{}\n\n## LOCAL LANGUAGE CODES:\n{}\n\n## "
"CONTEXT:\n{}",
location.city, location.country, local_language_codes,
safe_region_context);
const std::string retry_location =
std::format("City: {}, {}\nLocal language codes: {}", location.city,
location.country, local_language_codes);
std::string raw;
std::string last_error;
for (int attempt = 0; attempt < kMaxAttempts; ++attempt) {
raw = CallChatCompletionsApi(system_prompt, user_prompt,
kBreweryJsonSchema, "brewery_result",
kBreweryMaxTokens);
if (logger_) {
logger_->Log({.level = LogLevel::Debug,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"OpenAIGenerator: raw output (attempt {}): {}",
attempt + 1, raw)});
}
BreweryResult brewery;
const std::optional<std::string> validation_error =
ValidateBreweryJson(raw, brewery);
if (!validation_error.has_value()) {
if (logger_) {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"OpenAIGenerator: successfully generated "
"brewery data on attempt {}",
attempt + 1)});
}
return brewery;
}
last_error = *validation_error;
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"OpenAIGenerator: malformed brewery JSON (attempt {}): {}",
attempt + 1, *validation_error)});
}
user_prompt = std::format(
"Your previous response was invalid. Error: {}\nReturn real "
"content matching the required schema -- do not return "
"placeholder values.\n\n{}",
*validation_error, retry_location);
}
if (logger_) {
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"OpenAIGenerator: malformed brewery "
"response after {} attempts: {}",
kMaxAttempts, last_error.empty() ? raw : last_error)});
}
throw std::runtime_error("OpenAIGenerator: malformed brewery response");
}

114
tooling/pipeline/src/data_generation/openai/generate_user.cc Normal file
View File

@@ -0,0 +1,114 @@
/**
* @file data_generation/openai/generate_user.cc
* @brief Builds persona/name-grounded user prompts, calls the OpenAI Chat
* Completions API, and validates structured JSON output for user records.
*/
#include <format>
#include <optional>
#include <stdexcept>
#include <string>
#include "data_generation/generation_json_validation.h"
#include "data_generation/openai_generator.h"
#include "data_generation/openai_json_schemas.h"
namespace {
constexpr int kUserMaxTokens = 1200;
// Structured outputs already guarantee schema-valid JSON, so retries here
// are only for transient failures or placeholder text -- not JSON shape.
constexpr int kMaxAttempts = 2;
} // namespace
UserResult OpenAIGenerator::GenerateUser(const EnrichedCity& city,
const UserPersona& persona,
const Name& name) {
std::string style_affinities;
for (const std::string& style : persona.style_affinities) {
if (!style_affinities.empty()) {
style_affinities += ", ";
}
style_affinities += style;
}
const std::string system_prompt = prompt_directory_->Load("USER_GENERATION");
std::string user_prompt = std::format(
"## NAME:\n\n{} {}\n\n"
"## GENDER:\n\n{}\n\n"
"## CITY:\n\n{}\n\n"
"## COUNTRY:\n\n{}\n\n"
"## PERSONA:\n\n{}\n\n"
"## PERSONA DESCRIPTION:\n\n{}\n\n"
"## STYLE AFFINITIES:\n\n{}",
name.first_name, name.last_name, name.gender, city.location.city,
city.location.country, persona.name, persona.description,
style_affinities);
const std::string retry_context = std::format(
"Name: {} {}\nCity: {}, {}\nPersona: {}", name.first_name,
name.last_name, city.location.city, city.location.country,
persona.name);
std::string raw;
std::string last_error;
for (int attempt = 0; attempt < kMaxAttempts; ++attempt) {
raw = CallChatCompletionsApi(system_prompt, user_prompt, kUserJsonSchema,
"user_result", kUserMaxTokens);
if (logger_) {
logger_->Log({.level = LogLevel::Debug,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"OpenAIGenerator: raw output (attempt {}): {}",
attempt + 1, raw)});
}
UserResult user;
const std::optional<std::string> validation_error =
ValidateUserJson(raw, user);
if (!validation_error.has_value()) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::UserGeneration,
.message = std::format("OpenAIGenerator: successfully "
"generated user data on attempt {}",
attempt + 1)});
}
user.first_name = name.first_name;
user.last_name = name.last_name;
user.gender = name.gender;
return user;
}
last_error = *validation_error;
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"OpenAIGenerator: malformed user JSON (attempt {}): {}",
attempt + 1, *validation_error)});
}
user_prompt = std::format(
"Your previous response was invalid. Error: {}\nReturn real "
"content matching the required schema -- do not return "
"placeholder values.\n\n{}",
*validation_error, retry_context);
}
if (logger_) {
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"OpenAIGenerator: malformed user response "
"after {} attempts: {}",
kMaxAttempts, last_error.empty() ? raw : last_error)});
}
throw std::runtime_error("OpenAIGenerator: malformed user response");
}

20
tooling/pipeline/src/data_generation/openai/openai_generator.cc Normal file
View File

@@ -0,0 +1,20 @@
/**
* @file data_generation/openai/openai_generator.cc
* @brief OpenAIGenerator constructor.
*/
#include <memory>
#include <string>
#include <utility>
#include "data_generation/openai_generator.h"
OpenAIGenerator::OpenAIGenerator(
std::string api_key, std::string model, std::shared_ptr<ILogger> logger,
std::unique_ptr<IPromptDirectory> prompt_directory,
std::unique_ptr<WebClient> web_client)
: api_key_(std::move(api_key)),
model_(std::move(model)),
logger_(std::move(logger)),
prompt_directory_(std::move(prompt_directory)),
web_client_(std::move(web_client)) {}

58
tooling/pipeline/src/main.cc
View File

@@ -67,7 +67,7 @@ int main(const int argc, char** argv) {
std::unique_ptr<IPromptDirectory> prompt_directory;
if (!options.generator.use_mocked) {
if (options.generator.mode != GeneratorMode::kMock) {
try {
prompt_directory = std::make_unique<PromptDirectory>(
options.pipeline.prompt_dir, log_producer);
@@ -90,7 +90,7 @@ int main(const int argc, char** argv) {
di::bind<IPostalCodeService>().to(
[options,
&log_producer]() -> std::unique_ptr<IPostalCodeService> {
if (options.generator.use_mocked) {
if (options.generator.mode == GeneratorMode::kMock) {
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
@@ -110,13 +110,13 @@ int main(const int argc, char** argv) {
di::bind<ICuratedDataService>().to(
[options,
&log_producer]() -> std::unique_ptr<ICuratedDataService> {
if (options.generator.use_mocked) {
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Curated data: mock"});
return std::make_unique<MockCuratedDataService>();
}
// if (options.generator.mode == GeneratorMode::kMock) {
// log_producer->Log({.level = LogLevel::Info,
// .phase = PipelinePhase::Startup,
// .message = "Curated data: mock"});
//
// return std::make_unique<MockCuratedDataService>();
// }
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
@@ -131,13 +131,12 @@ int main(const int argc, char** argv) {
});
}),
di::bind<IPromptFormatter>().to([options, log_producer] {
if (options.generator.use_mocked) {
{
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Prompt formatter: none (mock mode)"});
}
if (options.generator.mode != GeneratorMode::kLlama) {
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Prompt formatter: none (not needed outside "
"Llama mode)"});
return std::unique_ptr<IPromptFormatter>(nullptr);
}
{
@@ -151,7 +150,7 @@ int main(const int argc, char** argv) {
std::make_unique<Gemma4JinjaPromptFormatter>());
}),
di::bind<WebClient>().to([options, log_producer] {
if (options.generator.use_mocked) {
if (options.generator.mode == GeneratorMode::kMock) {
{
log_producer->Log(
{.level = LogLevel::Info,
@@ -171,7 +170,7 @@ int main(const int argc, char** argv) {
di::bind<IEnrichmentService>().to(
[options, &log_producer](
const auto& inj) -> std::unique_ptr<IEnrichmentService> {
if (options.generator.use_mocked) {
if (options.generator.mode == GeneratorMode::kMock) {
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Enrichment: mock"});
@@ -191,7 +190,7 @@ int main(const int argc, char** argv) {
[&options, &model_path, &sampling, &prompt_directory,
&log_producer](
const auto& inj) -> std::unique_ptr<DataGenerator> {
if (options.generator.use_mocked) {
if (options.generator.mode == GeneratorMode::kMock) {
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Generator: mock"});
@@ -199,10 +198,25 @@ int main(const int argc, char** argv) {
return std::make_unique<MockGenerator>();
}
if (options.generator.mode == GeneratorMode::kOpenAI) {
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(
"Generator: OpenAIGenerator | model={}",
options.generator.openai_model)});
return std::make_unique<OpenAIGenerator>(
options.generator.openai_api_key,
options.generator.openai_model, log_producer,
std::move(prompt_directory),
inj.template create<std::unique_ptr<WebClient>>());
}
#ifdef BIERGARTEN_MOCK_ONLY
// Unreachable: ParseArguments requires --mocked when
// BIERGARTEN_MOCK_ONLY is compiled in, so LlamaGenerator is
// never constructed and is not linked into this binary.
// Only reachable when mode == Llama: Mock and OpenAI are
// handled above and don't need llama.cpp, but LlamaGenerator
// itself is not linked into a BIERGARTEN_MOCK_ONLY build.
throw std::runtime_error(
"LlamaGenerator is unavailable in a BIERGARTEN_MOCK_ONLY "
"build");

16
tooling/pipeline/src/services/curated_data/curated_json_data_service.cc
View File

@@ -30,6 +30,16 @@ static std::string ReadRequiredString(const boost::json::object& object,
return std::string(text);
}
static double ReadRequiredDouble(const boost::json::object& object,
const char* key) {
const boost::json::value* value = object.if_contains(key);
if (value == nullptr || !value->is_double()) {
throw std::runtime_error(
std::format("Missing or invalid double field: {}", key));
}
return value->as_double();
}
static std::vector<std::string> ReadRequiredStringArray(
const boost::json::object& object, const char* key) {
const boost::json::value* value = object.if_contains(key);
@@ -67,7 +77,6 @@ static PostalCodeSpec ReadPostalCodeSpec(const boost::json::object& object) {
}
namespace {
boost::json::value ParseJsonFile(const std::filesystem::path& filepath,
const char* what) {
std::ifstream input(filepath);
@@ -108,7 +117,6 @@ std::string ReadFirstOfStringArray(const boost::json::object& object,
throw std::runtime_error(
std::format("Missing or invalid string array field: {}", key));
}
} // namespace
CuratedJsonDataService::CuratedJsonDataService(CuratedDataFilePaths filepaths)
@@ -144,6 +152,8 @@ const LocationsList& CuratedJsonDataService::LoadLocations() {
.iso3166_2 = ReadRequiredString(object, "iso3166_2"),
.country = ReadRequiredString(object, "country"),
.iso3166_1 = ReadRequiredString(object, "iso3166_1"),
.latitude = ReadRequiredDouble(object, "latitude"),
.longitude = ReadRequiredDouble(object, "longitude"),
.local_languages = ReadRequiredStringArray(object, "local_languages"),
.postal_code = ReadPostalCodeSpec(object),
});
@@ -267,4 +277,4 @@ const SurnamesByCountryMap& CuratedJsonDataService::LoadSurnamesByCountry() {
cache_.surnames_by_country = std::move(surnames_by_country);
return cache_.surnames_by_country;
}
}

26
tooling/pipeline/src/services/sqlite/process_record.cc
View File

@@ -66,6 +66,18 @@ sqlite3_int64 SqliteExportService::ResolveCityId(const City& city) {
.index = sqlite_export_service_internal::kCityIso31661BindIndex,
.value = city.iso3166_1,
.action = "Failed to bind SQLite city ISO 3166-1 code"});
sqlite_export_service_internal::Bind(
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<double>{
.index = sqlite_export_service_internal::kCityLongitudeBindIndex,
.value = city.longitude,
.action = "Failed to bind SQLite city longitude"});
sqlite_export_service_internal::Bind(
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<double>{
.index = sqlite_export_service_internal::kCityLatitudeBindIndex,
.value = city.latitude,
.action = "Failed to bind SQLite city latitude"});
sqlite_export_service_internal::Bind(
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
@@ -165,6 +177,20 @@ uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
kBreweryAddressPostalCodeBindIndex,
.value = brewery.address.postal_code,
.action = "Failed to bind SQLite brewery address postal code"});
sqlite_export_service_internal::Bind(
insert_brewery_address_stmt_,
sqlite_export_service_internal::BoundParam<double>{
.index = sqlite_export_service_internal::
kBreweryAddressLongitudeBindIndex,
.value = brewery.address.longitude,
.action = "Failed to bind SQLite brewery address longitude"});
sqlite_export_service_internal::Bind(
insert_brewery_address_stmt_,
sqlite_export_service_internal::BoundParam<double>{
.index = sqlite_export_service_internal::
kBreweryAddressLatitudeBindIndex,
.value = brewery.address.latitude,
.action = "Failed to bind SQLite brewery address latitude"});
sqlite_export_service_internal::StepStatement(
db_handle_, insert_brewery_address_stmt_,

39
tooling/pipeline/src/web_client/http_web_client.cc
View File

@@ -13,6 +13,7 @@
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
#include "services/logging/logger.h"
@@ -67,6 +68,44 @@ std::string HttpWebClient::Get(const std::string& url) {
return result->body;
}
std::string HttpWebClient::Post(
const std::string& url, const std::string& body,
const std::vector<std::pair<std::string, std::string>>& headers) {
const auto [origin, path] = SplitUrl(url);
httplib::Client client(origin);
client.set_follow_location(true);
client.set_connection_timeout(kConnectionTimeoutSeconds);
client.set_read_timeout(kReadTimeoutSeconds);
httplib::Headers httplib_headers;
for (const auto& [name, value] : headers) {
httplib_headers.emplace(name, value);
}
const httplib::Result result =
client.Post(path, httplib_headers, body, "application/json");
if (!result) {
throw std::runtime_error(
std::format("[HttpWebClient] Request failed for URL: {} — {}", url,
httplib::to_string(result.error())));
}
if (result->status < kSuccessMin || result->status >= kSuccessMax) {
if (logger_) {
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"[HttpWebClient] Request failed for URL: {}", url)});
}
throw std::runtime_error(std::format(
"[HttpWebClient] HTTP {} for URL: {}", result->status, url));
}
return result->body;
}
std::string HttpWebClient::EncodeURL(const std::string& value) {
return httplib::encode_uri_component(value);
}