mirror of
https://github.com/aaronpo97/the-biergarten-app.git
synced 2026-10-15 01:54:43 +00:00
Update generate pipeline to add OpenAI api as an additional backend (for comparison testing), add brewery coordinates (#244)
* add brewery coordinates * Add openai mode * Add coordinates to schema * Fix C++ review findings in pipeline: k-prefixed enum values, include ordering, populated validation error messages, conventional const notation, and endl replaced with newline
This commit is contained in:
@@ -192,6 +192,19 @@ target_sources(${PROJECT_NAME} PRIVATE
|
|||||||
src/data_generation/mock/deterministic_hash.cc
|
src/data_generation/mock/deterministic_hash.cc
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# --- data_generation: shared JSON validation (always built — no llama.cpp dependency) ---
|
||||||
|
target_sources(${PROJECT_NAME} PRIVATE
|
||||||
|
src/data_generation/generation_json_validation.cc
|
||||||
|
)
|
||||||
|
|
||||||
|
# --- data_generation: openai (always built — no llama.cpp dependency) ---
|
||||||
|
target_sources(${PROJECT_NAME} PRIVATE
|
||||||
|
src/data_generation/openai/openai_generator.cc
|
||||||
|
src/data_generation/openai/call_chat_completions_api.cc
|
||||||
|
src/data_generation/openai/generate_brewery.cc
|
||||||
|
src/data_generation/openai/generate_user.cc
|
||||||
|
)
|
||||||
|
|
||||||
# --- data_generation: llama (skipped for mock-only builds) ---
|
# --- data_generation: llama (skipped for mock-only builds) ---
|
||||||
if(NOT BIERGARTEN_MOCK_ONLY)
|
if(NOT BIERGARTEN_MOCK_ONLY)
|
||||||
target_sources(${PROJECT_NAME} PRIVATE
|
target_sources(${PROJECT_NAME} PRIVATE
|
||||||
|
|||||||
@@ -15,12 +15,15 @@
|
|||||||
|
|
||||||
// --- data_generation ---
|
// --- data_generation ---
|
||||||
#include "data_generation/data_generator.h"
|
#include "data_generation/data_generator.h"
|
||||||
|
#include "data_generation/generation_json_validation.h"
|
||||||
#include "data_generation/json_grammars.h"
|
#include "data_generation/json_grammars.h"
|
||||||
#ifndef BIERGARTEN_MOCK_ONLY
|
#ifndef BIERGARTEN_MOCK_ONLY
|
||||||
#include "data_generation/llama_generator.h"
|
#include "data_generation/llama_generator.h"
|
||||||
#include "data_generation/llama_generator_helpers.h"
|
#include "data_generation/llama_generator_helpers.h"
|
||||||
#endif
|
#endif
|
||||||
#include "data_generation/mock_generator.h"
|
#include "data_generation/mock_generator.h"
|
||||||
|
#include "data_generation/openai_generator.h"
|
||||||
|
#include "data_generation/openai_json_schemas.h"
|
||||||
#include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h"
|
#include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h"
|
||||||
#include "data_generation/prompt_formatting/prompt_formatter.h"
|
#include "data_generation/prompt_formatting/prompt_formatter.h"
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,52 @@
|
|||||||
|
#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_GENERATION_JSON_VALIDATION_H_
|
||||||
|
#define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_GENERATION_JSON_VALIDATION_H_
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @file data_generation/generation_json_validation.h
|
||||||
|
* @brief Generator-agnostic helpers for preparing prompt context and
|
||||||
|
* validating structured JSON output. Shared across all DataGenerator
|
||||||
|
* implementations (Llama, OpenAI, ...).
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <cstddef>
|
||||||
|
#include <optional>
|
||||||
|
#include <string>
|
||||||
|
#include <string_view>
|
||||||
|
|
||||||
|
#include "data_model/generated_models.h"
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Normalizes and truncates regional context.
|
||||||
|
*
|
||||||
|
* @param region_context Input regional context text.
|
||||||
|
* @param max_chars Maximum output length.
|
||||||
|
* @return Processed region context.
|
||||||
|
*/
|
||||||
|
std::string PrepareRegionContext(std::string_view region_context,
|
||||||
|
size_t max_chars = 2000);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Validates and parses brewery JSON output.
|
||||||
|
*
|
||||||
|
* @param raw Raw model output.
|
||||||
|
* @param brewery_out Parsed brewery payload.
|
||||||
|
* @return Validation error message if invalid, or std::nullopt on success.
|
||||||
|
*/
|
||||||
|
std::optional<std::string> ValidateBreweryJson(const std::string& raw,
|
||||||
|
BreweryResult& brewery_out);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Validates and parses user JSON output.
|
||||||
|
*
|
||||||
|
* Only populates `username`, `bio`, and `activity_weight` -- `first_name`
|
||||||
|
* and `last_name` are not LLM-authored and are set separately from the
|
||||||
|
* sampled Name.
|
||||||
|
*
|
||||||
|
* @param raw Raw model output.
|
||||||
|
* @param user_out Parsed user payload.
|
||||||
|
* @return Validation error message if invalid, or std::nullopt on success.
|
||||||
|
*/
|
||||||
|
std::optional<std::string> ValidateUserJson(const std::string& raw,
|
||||||
|
UserResult& user_out);
|
||||||
|
|
||||||
|
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_GENERATION_JSON_VALIDATION_H_
|
||||||
@@ -3,30 +3,17 @@
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* @file data_generation/llama_generator_helpers.h
|
* @file data_generation/llama_generator_helpers.h
|
||||||
* @brief Shared helper APIs used by LlamaGenerator translation units.
|
* @brief Llama.cpp-specific helper APIs used by LlamaGenerator translation
|
||||||
|
* units. Generator-agnostic helpers (prompt-context prep, JSON validation)
|
||||||
|
* live in data_generation/generation_json_validation.h instead.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include <cstddef>
|
|
||||||
#include <cstdint>
|
#include <cstdint>
|
||||||
#include <optional>
|
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <string_view>
|
|
||||||
|
|
||||||
#include "data_model/generated_models.h"
|
|
||||||
|
|
||||||
struct llama_vocab;
|
struct llama_vocab;
|
||||||
using llama_token = int32_t;
|
using llama_token = int32_t;
|
||||||
|
|
||||||
/**
|
|
||||||
* @brief Normalizes and truncates regional context.
|
|
||||||
*
|
|
||||||
* @param region_context Input regional context text.
|
|
||||||
* @param max_chars Maximum output length.
|
|
||||||
* @return Processed region context.
|
|
||||||
*/
|
|
||||||
std::string PrepareRegionContext(std::string_view region_context,
|
|
||||||
size_t max_chars = 2000);
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Decodes a sampled token and appends it to output text.
|
* @brief Decodes a sampled token and appends it to output text.
|
||||||
*
|
*
|
||||||
@@ -37,28 +24,4 @@ std::string PrepareRegionContext(std::string_view region_context,
|
|||||||
void AppendTokenPiece(const llama_vocab* vocab, llama_token token,
|
void AppendTokenPiece(const llama_vocab* vocab, llama_token token,
|
||||||
std::string& output);
|
std::string& output);
|
||||||
|
|
||||||
/**
|
|
||||||
* @brief Validates and parses brewery JSON output.
|
|
||||||
*
|
|
||||||
* @param raw Raw model output.
|
|
||||||
* @param brewery_out Parsed brewery payload.
|
|
||||||
* @return Validation error message if invalid, or std::nullopt on success.
|
|
||||||
*/
|
|
||||||
std::optional<std::string> ValidateBreweryJson(const std::string& raw,
|
|
||||||
BreweryResult& brewery_out);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @brief Validates and parses user JSON output.
|
|
||||||
*
|
|
||||||
* Only populates `username`, `bio`, and `activity_weight` -- `first_name`
|
|
||||||
* and `last_name` are not LLM-authored and are set separately from the
|
|
||||||
* sampled Name.
|
|
||||||
*
|
|
||||||
* @param raw Raw model output.
|
|
||||||
* @param user_out Parsed user payload.
|
|
||||||
* @return Validation error message if invalid, or std::nullopt on success.
|
|
||||||
*/
|
|
||||||
std::optional<std::string> ValidateUserJson(const std::string& raw,
|
|
||||||
UserResult& user_out);
|
|
||||||
|
|
||||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_HELPERS_H_
|
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_HELPERS_H_
|
||||||
|
|||||||
104
tooling/pipeline/includes/data_generation/openai_generator.h
Normal file
104
tooling/pipeline/includes/data_generation/openai_generator.h
Normal file
@@ -0,0 +1,104 @@
|
|||||||
|
#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_OPENAI_GENERATOR_H_
|
||||||
|
#define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_OPENAI_GENERATOR_H_
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @file data_generation/openai_generator.h
|
||||||
|
* @brief OpenAI Chat Completions API-backed implementation of DataGenerator.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <memory>
|
||||||
|
#include <string>
|
||||||
|
#include <string_view>
|
||||||
|
|
||||||
|
#include "data_generation/data_generator.h"
|
||||||
|
#include "data_model/models.h"
|
||||||
|
#include "services/logging/logger.h"
|
||||||
|
#include "services/prompting/prompt_directory.h"
|
||||||
|
#include "web_client/web_client.h"
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Data generator implementation backed by the OpenAI Chat
|
||||||
|
* Completions API.
|
||||||
|
*
|
||||||
|
* Unlike LlamaGenerator, this generator has no local model state and no
|
||||||
|
* chat-template formatting step -- Chat Completions takes system/user
|
||||||
|
* messages directly, and Structured Outputs (response_format: json_schema,
|
||||||
|
* strict: true) guarantee schema-valid JSON, replacing llama.cpp's GBNF
|
||||||
|
* grammar constraint.
|
||||||
|
*/
|
||||||
|
class OpenAIGenerator final : public DataGenerator {
|
||||||
|
public:
|
||||||
|
/**
|
||||||
|
* @brief Constructs a generator that calls the OpenAI Chat Completions
|
||||||
|
* API.
|
||||||
|
*
|
||||||
|
* @param api_key OpenAI API key (from the OPENAI_API_KEY environment
|
||||||
|
* variable -- never accepted as a CLI argument).
|
||||||
|
* @param model OpenAI model ID, e.g. "gpt-4o-mini".
|
||||||
|
* @param logger Sink for pipeline diagnostics.
|
||||||
|
* @param prompt_directory Directory service for loading named prompt
|
||||||
|
* files (the same BREWERY_GENERATION/USER_GENERATION prompts Llama
|
||||||
|
* uses -- they're backend-agnostic).
|
||||||
|
* @param web_client HTTP client used to reach the Chat Completions API.
|
||||||
|
*/
|
||||||
|
OpenAIGenerator(std::string api_key, std::string model,
|
||||||
|
std::shared_ptr<ILogger> logger,
|
||||||
|
std::unique_ptr<IPromptDirectory> prompt_directory,
|
||||||
|
std::unique_ptr<WebClient> web_client);
|
||||||
|
|
||||||
|
~OpenAIGenerator() override = default;
|
||||||
|
|
||||||
|
OpenAIGenerator(const OpenAIGenerator&) = delete;
|
||||||
|
OpenAIGenerator& operator=(const OpenAIGenerator&) = delete;
|
||||||
|
OpenAIGenerator(OpenAIGenerator&&) = delete;
|
||||||
|
OpenAIGenerator& operator=(OpenAIGenerator&&) = delete;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Generates brewery data for a specific location.
|
||||||
|
*
|
||||||
|
* @param enriched_city Enriched city the brewery is associated with.
|
||||||
|
* @return Generated brewery result.
|
||||||
|
*/
|
||||||
|
BreweryResult GenerateBrewery(const EnrichedCity& enriched_city) override;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Generates a user profile grounded in a sampled name and persona.
|
||||||
|
*
|
||||||
|
* @param city Enriched city the user is associated with.
|
||||||
|
* @param persona Persona archetype grounding the generated bio.
|
||||||
|
* @param name Sampled first/last name -- not LLM-invented.
|
||||||
|
* @return Generated user profile.
|
||||||
|
*/
|
||||||
|
UserResult GenerateUser(const EnrichedCity& city, const UserPersona& persona,
|
||||||
|
const Name& name) override;
|
||||||
|
|
||||||
|
private:
|
||||||
|
/**
|
||||||
|
* @brief Sends a single Chat Completions request with a Structured
|
||||||
|
* Outputs schema and returns the raw JSON text of the response.
|
||||||
|
*
|
||||||
|
* @param system_prompt System role message content.
|
||||||
|
* @param user_prompt User role message content.
|
||||||
|
* @param json_schema Raw JSON Schema text constraining the response
|
||||||
|
* shape (see data_generation/openai_json_schemas.h).
|
||||||
|
* @param schema_name Identifier for the schema, required by the
|
||||||
|
* response_format.json_schema wrapper.
|
||||||
|
* @param max_tokens Maximum completion tokens to generate.
|
||||||
|
* @return Raw JSON text produced by the model.
|
||||||
|
* @throws std::runtime_error on a request/HTTP failure, a safety-policy
|
||||||
|
* refusal, or a response with no message content.
|
||||||
|
*/
|
||||||
|
std::string CallChatCompletionsApi(const std::string& system_prompt,
|
||||||
|
const std::string& user_prompt,
|
||||||
|
std::string_view json_schema,
|
||||||
|
std::string_view schema_name,
|
||||||
|
int max_tokens);
|
||||||
|
|
||||||
|
std::string api_key_;
|
||||||
|
std::string model_;
|
||||||
|
std::shared_ptr<ILogger> logger_;
|
||||||
|
std::unique_ptr<IPromptDirectory> prompt_directory_;
|
||||||
|
std::unique_ptr<WebClient> web_client_;
|
||||||
|
};
|
||||||
|
|
||||||
|
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_OPENAI_GENERATOR_H_
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_OPENAI_JSON_SCHEMAS_H_
|
||||||
|
#define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_OPENAI_JSON_SCHEMAS_H_
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @file data_generation/openai_json_schemas.h
|
||||||
|
* @brief JSON Schemas constraining structured output from OpenAIGenerator
|
||||||
|
* Chat Completions requests, via response_format.json_schema (Structured
|
||||||
|
* Outputs, strict mode). This is OpenAI's equivalent of the GBNF grammars
|
||||||
|
* in json_grammars.h -- the API guarantees the response message content is
|
||||||
|
* a single JSON string that validates against the schema, so there's no
|
||||||
|
* grammar-level "thought-block" prefix to account for.
|
||||||
|
*
|
||||||
|
* OpenAI's strict mode requires "additionalProperties": false on every
|
||||||
|
* object in the schema, which these already have.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <string_view>
|
||||||
|
|
||||||
|
inline constexpr std::string_view kBreweryJsonSchema = R"json_brewery(
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"name_en": {"type": "string"},
|
||||||
|
"description_en": {"type": "string"},
|
||||||
|
"name_local": {"type": "string"},
|
||||||
|
"description_local": {"type": "string"}
|
||||||
|
},
|
||||||
|
"required": ["name_en", "description_en", "name_local", "description_local"],
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
|
)json_brewery";
|
||||||
|
|
||||||
|
inline constexpr std::string_view kUserJsonSchema = R"json_user(
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"username": {"type": "string"},
|
||||||
|
"bio": {"type": "string"},
|
||||||
|
"activity_weight": {"type": "number"}
|
||||||
|
},
|
||||||
|
"required": ["username", "bio", "activity_weight"],
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
|
)json_user";
|
||||||
|
|
||||||
|
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_OPENAI_JSON_SCHEMAS_H_
|
||||||
@@ -94,6 +94,8 @@ struct EnrichedCity {
|
|||||||
struct BreweryAddress {
|
struct BreweryAddress {
|
||||||
City city;
|
City city;
|
||||||
std::string postal_code{};
|
std::string postal_code{};
|
||||||
|
double longitude{};
|
||||||
|
double latitude{};
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -66,6 +66,9 @@ struct City {
|
|||||||
*/
|
*/
|
||||||
std::string iso3166_1{};
|
std::string iso3166_1{};
|
||||||
|
|
||||||
|
double longitude{};
|
||||||
|
double latitude{};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Local language codes in priority order.
|
* @brief Local language codes in priority order.
|
||||||
*/
|
*/
|
||||||
@@ -184,25 +187,47 @@ struct SamplingOptions {
|
|||||||
int n_gpu_layers = 0;
|
int n_gpu_layers = 0;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Selects which DataGenerator backend the pipeline uses.
|
||||||
|
*/
|
||||||
|
enum class GeneratorMode {
|
||||||
|
kMock,
|
||||||
|
kLlama,
|
||||||
|
kOpenAI,
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Configuration for the LLM generator component.
|
* @brief Configuration for the LLM generator component.
|
||||||
*/
|
*/
|
||||||
struct GeneratorOptions {
|
struct GeneratorOptions {
|
||||||
/**
|
/**
|
||||||
* @brief Path to the LLM model file (gguf format).
|
* @brief Path to the LLM model file (gguf format). Only meaningful for
|
||||||
|
* GeneratorMode::kLlama.
|
||||||
*/
|
*/
|
||||||
std::filesystem::path model_path;
|
std::filesystem::path model_path;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Use mocked generator instead of actual LLM inference.
|
* @brief Which generator backend to use.
|
||||||
*/
|
*/
|
||||||
bool use_mocked = false;
|
GeneratorMode mode = GeneratorMode::kMock;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Specific sampling parameters for this generator.
|
* @brief Specific sampling parameters for this generator. Only
|
||||||
* If nullopt, the application should use global defaults.
|
* meaningful for GeneratorMode::kLlama; if nullopt, LlamaGenerator uses
|
||||||
|
* its own defaults.
|
||||||
*/
|
*/
|
||||||
std::optional<SamplingOptions> sampling;
|
std::optional<SamplingOptions> sampling;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief OpenAI API key, read from the OPENAI_API_KEY environment
|
||||||
|
* variable. Only populated for GeneratorMode::kOpenAI.
|
||||||
|
*/
|
||||||
|
std::string openai_api_key;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief OpenAI model ID used by OpenAIGenerator.
|
||||||
|
*/
|
||||||
|
std::string openai_model = "gpt-4o-mini";
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -26,6 +26,8 @@ CREATE TABLE IF NOT EXISTS cities (
|
|||||||
iso3166_2 TEXT NOT NULL,
|
iso3166_2 TEXT NOT NULL,
|
||||||
country TEXT NOT NULL,
|
country TEXT NOT NULL,
|
||||||
iso3166_1 TEXT NOT NULL,
|
iso3166_1 TEXT NOT NULL,
|
||||||
|
longitude REAL NOT NULL,
|
||||||
|
latitude REAL NOT NULL,
|
||||||
local_languages_json TEXT NOT NULL,
|
local_languages_json TEXT NOT NULL,
|
||||||
postal_code_country_format_regex TEXT NOT NULL,
|
postal_code_country_format_regex TEXT NOT NULL,
|
||||||
postal_code_city_regex_json TEXT NOT NULL,
|
postal_code_city_regex_json TEXT NOT NULL,
|
||||||
@@ -53,6 +55,8 @@ CREATE TABLE IF NOT EXISTS brewery_addresses (
|
|||||||
brewery_id INTEGER NOT NULL,
|
brewery_id INTEGER NOT NULL,
|
||||||
city_id INTEGER NOT NULL,
|
city_id INTEGER NOT NULL,
|
||||||
postal_code TEXT NOT NULL,
|
postal_code TEXT NOT NULL,
|
||||||
|
longitude REAL NOT NULL,
|
||||||
|
latitude REAL NOT NULL,
|
||||||
FOREIGN KEY(brewery_id) REFERENCES breweries(id) ON DELETE CASCADE,
|
FOREIGN KEY(brewery_id) REFERENCES breweries(id) ON DELETE CASCADE,
|
||||||
FOREIGN KEY(city_id) REFERENCES cities(id) ON DELETE CASCADE
|
FOREIGN KEY(city_id) REFERENCES cities(id) ON DELETE CASCADE
|
||||||
);
|
);
|
||||||
@@ -105,10 +109,12 @@ INSERT INTO cities (
|
|||||||
iso3166_2,
|
iso3166_2,
|
||||||
country,
|
country,
|
||||||
iso3166_1,
|
iso3166_1,
|
||||||
|
longitude,
|
||||||
|
latitude,
|
||||||
local_languages_json,
|
local_languages_json,
|
||||||
postal_code_country_format_regex,
|
postal_code_country_format_regex,
|
||||||
postal_code_city_regex_json
|
postal_code_city_regex_json
|
||||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?);
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
|
||||||
)sql";
|
)sql";
|
||||||
|
|
||||||
inline constexpr std::string_view kInsertBrewerySql = R"sql(
|
inline constexpr std::string_view kInsertBrewerySql = R"sql(
|
||||||
@@ -124,8 +130,10 @@ inline constexpr std::string_view kInsertBreweryAddressSql = R"sql(
|
|||||||
INSERT INTO brewery_addresses (
|
INSERT INTO brewery_addresses (
|
||||||
brewery_id,
|
brewery_id,
|
||||||
city_id,
|
city_id,
|
||||||
postal_code
|
postal_code,
|
||||||
) VALUES (?, ?, ?);
|
longitude,
|
||||||
|
latitude
|
||||||
|
) VALUES (?, ?, ?, ?, ?);
|
||||||
)sql";
|
)sql";
|
||||||
|
|
||||||
inline constexpr std::string_view kInsertUserSql = R"sql(
|
inline constexpr std::string_view kInsertUserSql = R"sql(
|
||||||
@@ -157,6 +165,8 @@ enum CityBindIndex {
|
|||||||
kCityIso31662BindIndex,
|
kCityIso31662BindIndex,
|
||||||
kCityCountryBindIndex,
|
kCityCountryBindIndex,
|
||||||
kCityIso31661BindIndex,
|
kCityIso31661BindIndex,
|
||||||
|
kCityLongitudeBindIndex,
|
||||||
|
kCityLatitudeBindIndex,
|
||||||
kCityLanguagesBindIndex,
|
kCityLanguagesBindIndex,
|
||||||
kCityPostalCodeCountryFormatRegexBindIndex,
|
kCityPostalCodeCountryFormatRegexBindIndex,
|
||||||
kCityPostalCodeCityRegexJsonBindIndex,
|
kCityPostalCodeCityRegexJsonBindIndex,
|
||||||
@@ -173,6 +183,8 @@ enum BreweryAddressBindIndex {
|
|||||||
kBreweryAddressBreweryIdBindIndex = 1,
|
kBreweryAddressBreweryIdBindIndex = 1,
|
||||||
kBreweryAddressCityIdBindIndex,
|
kBreweryAddressCityIdBindIndex,
|
||||||
kBreweryAddressPostalCodeBindIndex,
|
kBreweryAddressPostalCodeBindIndex,
|
||||||
|
kBreweryAddressLongitudeBindIndex,
|
||||||
|
kBreweryAddressLatitudeBindIndex,
|
||||||
};
|
};
|
||||||
|
|
||||||
enum UserBindIndex {
|
enum UserBindIndex {
|
||||||
|
|||||||
@@ -9,6 +9,7 @@
|
|||||||
#include <memory>
|
#include <memory>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <utility>
|
#include <utility>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
#include "services/logging/logger.h"
|
#include "services/logging/logger.h"
|
||||||
#include "web_client/web_client.h"
|
#include "web_client/web_client.h"
|
||||||
@@ -39,6 +40,22 @@ class HttpWebClient final : public WebClient {
|
|||||||
*/
|
*/
|
||||||
std::string Get(const std::string& url) override;
|
std::string Get(const std::string& url) override;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Executes a blocking HTTP/HTTPS POST request against a full URL.
|
||||||
|
*
|
||||||
|
* Sends @p body with a "Content-Type: application/json" header, plus any
|
||||||
|
* additional headers supplied by the caller (e.g. authentication).
|
||||||
|
*
|
||||||
|
* @param url Fully-qualified URL.
|
||||||
|
* @param body Request body, sent as-is.
|
||||||
|
* @param headers Additional request headers.
|
||||||
|
* @return Response body on HTTP 2xx; throws std::runtime_error otherwise.
|
||||||
|
*/
|
||||||
|
std::string Post(
|
||||||
|
const std::string& url, const std::string& body,
|
||||||
|
const std::vector<std::pair<std::string, std::string>>& headers)
|
||||||
|
override;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Percent-encodes a single URI component (query parameter value or
|
* @brief Percent-encodes a single URI component (query parameter value or
|
||||||
* path segment). Delegates to httplib::encode_uri_component().
|
* path segment). Delegates to httplib::encode_uri_component().
|
||||||
|
|||||||
@@ -7,6 +7,8 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
#include <string>
|
#include <string>
|
||||||
|
#include <utility>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Abstract web client interface.
|
* @brief Abstract web client interface.
|
||||||
@@ -23,6 +25,20 @@ class WebClient {
|
|||||||
*/
|
*/
|
||||||
virtual std::string Get(const std::string& url) = 0;
|
virtual std::string Get(const std::string& url) = 0;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Executes an HTTP POST request with a JSON body.
|
||||||
|
*
|
||||||
|
* @param url Request URL.
|
||||||
|
* @param body Request body (sent as-is; callers are responsible for
|
||||||
|
* providing valid JSON when the target API expects it).
|
||||||
|
* @param headers Additional request headers (e.g. authentication),
|
||||||
|
* sent alongside a fixed "Content-Type: application/json" header.
|
||||||
|
* @return Response body.
|
||||||
|
*/
|
||||||
|
virtual std::string Post(
|
||||||
|
const std::string& url, const std::string& body,
|
||||||
|
const std::vector<std::pair<std::string, std::string>>& headers) = 0;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief URL-encodes a string value.
|
* @brief URL-encodes a string value.
|
||||||
*
|
*
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
#include <chrono>
|
#include <chrono>
|
||||||
|
#include <cstdlib>
|
||||||
#include <format>
|
#include <format>
|
||||||
#include <iostream>
|
#include <iostream>
|
||||||
#include <optional>
|
#include <optional>
|
||||||
@@ -37,13 +38,20 @@ std::optional<ApplicationOptions> ParseArguments(
|
|||||||
"Number of layers to offload to GPU");
|
"Number of layers to offload to GPU");
|
||||||
};
|
};
|
||||||
|
|
||||||
// --mocked and --model are mutually exclusive; validation is enforced below
|
// --mocked, --model, and --openai are mutually exclusive; validation is
|
||||||
// rather than at registration to produce a clear diagnostic message.
|
// enforced below rather than at registration to produce a clear
|
||||||
|
// diagnostic message.
|
||||||
auto add_generator_options = [&]() -> void {
|
auto add_generator_options = [&]() -> void {
|
||||||
opt("mocked", prog_opts::bool_switch(),
|
opt("mocked", prog_opts::bool_switch(),
|
||||||
"Use mocked generator for brewery/user data");
|
"Use mocked generator for brewery/user data");
|
||||||
opt("model,m", prog_opts::value<std::string>()->default_value(""),
|
opt("model,m", prog_opts::value<std::string>()->default_value(""),
|
||||||
"Path to LLM model (gguf)");
|
"Path to LLM model (gguf)");
|
||||||
|
opt("openai", prog_opts::bool_switch(),
|
||||||
|
"Use the OpenAI API (Chat Completions) for brewery/user data. "
|
||||||
|
"Requires the OPENAI_API_KEY environment variable.");
|
||||||
|
opt("openai-model",
|
||||||
|
prog_opts::value<std::string>()->default_value("gpt-4o-mini"),
|
||||||
|
"OpenAI model ID used when --openai is set");
|
||||||
};
|
};
|
||||||
|
|
||||||
auto add_pipeline_options = [&]() -> void {
|
auto add_pipeline_options = [&]() -> void {
|
||||||
@@ -109,14 +117,19 @@ std::optional<ApplicationOptions> ParseArguments(
|
|||||||
|
|
||||||
const bool use_mocked = var_map["mocked"].as<bool>();
|
const bool use_mocked = var_map["mocked"].as<bool>();
|
||||||
const std::string model_path = var_map["model"].as<std::string>();
|
const std::string model_path = var_map["model"].as<std::string>();
|
||||||
|
const bool use_openai = var_map["openai"].as<bool>();
|
||||||
const int n_gpu_layers = var_map["n-gpu-layers"].as<int>();
|
const int n_gpu_layers = var_map["n-gpu-layers"].as<int>();
|
||||||
|
|
||||||
// Enforce mutual exclusivity before any further configuration is
|
// Enforce mutual exclusivity before any further configuration is
|
||||||
// applied.
|
// applied.
|
||||||
if (use_mocked && !model_path.empty()) {
|
const int selected_generator_count =
|
||||||
|
(use_mocked ? 1 : 0) + (!model_path.empty() ? 1 : 0) +
|
||||||
|
(use_openai ? 1 : 0);
|
||||||
|
|
||||||
|
if (selected_generator_count > 1) {
|
||||||
const std::string msg =
|
const std::string msg =
|
||||||
"Invalid arguments: --mocked and --model are mutually "
|
"Invalid arguments: --mocked, --model, and --openai are "
|
||||||
"exclusive";
|
"mutually exclusive";
|
||||||
if (logger) {
|
if (logger) {
|
||||||
logger->Log(LogDTO{.level = LogLevel::Error,
|
logger->Log(LogDTO{.level = LogLevel::Error,
|
||||||
.phase = PipelinePhase::Startup,
|
.phase = PipelinePhase::Startup,
|
||||||
@@ -127,10 +140,10 @@ std::optional<ApplicationOptions> ParseArguments(
|
|||||||
return std::nullopt;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!use_mocked && model_path.empty()) {
|
if (selected_generator_count == 0) {
|
||||||
const std::string msg =
|
const std::string msg =
|
||||||
"Invalid arguments: either --mocked or --model must be "
|
"Invalid arguments: exactly one of --mocked, --model, or "
|
||||||
"specified";
|
"--openai must be specified";
|
||||||
if (logger) {
|
if (logger) {
|
||||||
logger->Log(LogDTO{.level = LogLevel::Error,
|
logger->Log(LogDTO{.level = LogLevel::Error,
|
||||||
.phase = PipelinePhase::Startup,
|
.phase = PipelinePhase::Startup,
|
||||||
@@ -141,8 +154,9 @@ std::optional<ApplicationOptions> ParseArguments(
|
|||||||
return std::nullopt;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prompt directory is only meaningful for live inference — the mock
|
// Prompt directory is only meaningful for live inference (Llama or
|
||||||
// generator has no use for it and should not require it to be present.
|
// OpenAI) — the mock generator has no use for it and should not
|
||||||
|
// require it to be present.
|
||||||
if (!use_mocked && options.pipeline.prompt_dir.empty()) {
|
if (!use_mocked && options.pipeline.prompt_dir.empty()) {
|
||||||
const std::string msg =
|
const std::string msg =
|
||||||
"Invalid arguments: --prompt-dir is required when not using "
|
"Invalid arguments: --prompt-dir is required when not using "
|
||||||
@@ -157,8 +171,29 @@ std::optional<ApplicationOptions> ParseArguments(
|
|||||||
return std::nullopt;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
options.generator.use_mocked = use_mocked;
|
if (use_openai) {
|
||||||
|
const char* api_key_env = std::getenv("OPENAI_API_KEY");
|
||||||
|
if (api_key_env == nullptr || api_key_env[0] == '\0') {
|
||||||
|
const std::string msg =
|
||||||
|
"Invalid arguments: the OPENAI_API_KEY environment "
|
||||||
|
"variable must be set when using --openai";
|
||||||
|
if (logger) {
|
||||||
|
logger->Log({.level = LogLevel::Error,
|
||||||
|
.phase = PipelinePhase::Startup,
|
||||||
|
.message = msg});
|
||||||
|
} else {
|
||||||
|
std::cerr << msg << '\n';
|
||||||
|
}
|
||||||
|
return std::nullopt;
|
||||||
|
}
|
||||||
|
options.generator.openai_api_key = api_key_env;
|
||||||
|
}
|
||||||
|
|
||||||
|
options.generator.mode = use_mocked ? GeneratorMode::kMock
|
||||||
|
: use_openai ? GeneratorMode::kOpenAI
|
||||||
|
: GeneratorMode::kLlama;
|
||||||
options.generator.model_path = model_path;
|
options.generator.model_path = model_path;
|
||||||
|
options.generator.openai_model = var_map["openai-model"].as<std::string>();
|
||||||
|
|
||||||
// Only populate sampling config when the user explicitly overrides at
|
// Only populate sampling config when the user explicitly overrides at
|
||||||
// least one value. Leaving it as std::nullopt lets LlamaGenerator fall
|
// least one value. Leaving it as std::nullopt lets LlamaGenerator fall
|
||||||
@@ -172,10 +207,11 @@ std::optional<ApplicationOptions> ParseArguments(
|
|||||||
|
|
||||||
if (user_provided_sampling) {
|
if (user_provided_sampling) {
|
||||||
// Warn but do not fail — the run is still valid, the flags are just
|
// Warn but do not fail — the run is still valid, the flags are just
|
||||||
// silently irrelevant when no model is loaded.
|
// silently irrelevant when no local model is loaded.
|
||||||
if (use_mocked) {
|
if (options.generator.mode != GeneratorMode::kLlama) {
|
||||||
const std::string msg =
|
const std::string msg =
|
||||||
"Sampling parameters are ignored when using --mocked";
|
"Sampling parameters are ignored unless using --model "
|
||||||
|
"(Llama)";
|
||||||
if (logger) {
|
if (logger) {
|
||||||
logger->Log(LogDTO{.level = LogLevel::Warn,
|
logger->Log(LogDTO{.level = LogLevel::Warn,
|
||||||
.phase = PipelinePhase::Startup,
|
.phase = PipelinePhase::Startup,
|
||||||
|
|||||||
@@ -4,12 +4,55 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
#include <chrono>
|
#include <chrono>
|
||||||
|
#include <cmath>
|
||||||
#include <format>
|
#include <format>
|
||||||
|
#include <numbers>
|
||||||
#include <optional>
|
#include <optional>
|
||||||
|
#include <random>
|
||||||
|
|
||||||
#include "biergarten_pipeline_orchestrator.h"
|
#include "biergarten_pipeline_orchestrator.h"
|
||||||
#include "services/logging/logger.h"
|
#include "services/logging/logger.h"
|
||||||
|
|
||||||
|
struct Coords {
|
||||||
|
double longitude;
|
||||||
|
double latitude;
|
||||||
|
};
|
||||||
|
|
||||||
|
// @todo find a better home for this
|
||||||
|
namespace {
|
||||||
|
constexpr double kEarthRadiusKm = 6371.0;
|
||||||
|
|
||||||
|
double DegreesToRadians(double degrees) {
|
||||||
|
return degrees * std::numbers::pi / 180.0;
|
||||||
|
}
|
||||||
|
|
||||||
|
double RadiansToDegrees(double radians) {
|
||||||
|
return radians * 180.0 / std::numbers::pi;
|
||||||
|
}
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
static Coords GetRandomCoordsWithinRange(const Coords& centre,
|
||||||
|
const int distance_km,
|
||||||
|
std::mt19937& rng) {
|
||||||
|
std::uniform_real_distribution<double> angle_dist(0.0,
|
||||||
|
2.0 * std::numbers::pi);
|
||||||
|
std::uniform_real_distribution<double> unit_dist(0.0, 1.0);
|
||||||
|
|
||||||
|
const double angle_radians = angle_dist(rng);
|
||||||
|
// sqrt() keeps points uniformly distributed over the disc's area rather
|
||||||
|
// than clustering near the centre.
|
||||||
|
const double radius_km = distance_km * std::sqrt(unit_dist(rng));
|
||||||
|
|
||||||
|
const double delta_latitude_degrees =
|
||||||
|
RadiansToDegrees(radius_km * std::cos(angle_radians) / kEarthRadiusKm);
|
||||||
|
const double delta_longitude_degrees = RadiansToDegrees(
|
||||||
|
radius_km * std::sin(angle_radians) /
|
||||||
|
(kEarthRadiusKm * std::cos(DegreesToRadians(centre.latitude))));
|
||||||
|
|
||||||
|
return Coords{.longitude = centre.longitude + delta_longitude_degrees,
|
||||||
|
.latitude = centre.latitude + delta_latitude_degrees};
|
||||||
|
}
|
||||||
|
|
||||||
void BiergartenPipelineOrchestrator::GenerateBreweries(
|
void BiergartenPipelineOrchestrator::GenerateBreweries(
|
||||||
std::span<const EnrichedCity> cities) {
|
std::span<const EnrichedCity> cities) {
|
||||||
logger_->Log({.level = LogLevel::Info,
|
logger_->Log({.level = LogLevel::Info,
|
||||||
@@ -19,18 +62,28 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
|
|||||||
generated_breweries_.clear();
|
generated_breweries_.clear();
|
||||||
size_t skipped_count = 0;
|
size_t skipped_count = 0;
|
||||||
size_t export_failed_count = 0;
|
size_t export_failed_count = 0;
|
||||||
|
std::mt19937 rng(std::random_device{}());
|
||||||
|
|
||||||
const auto generate_record =
|
const auto generate_record =
|
||||||
[this, &skipped_count](
|
[this, &skipped_count, &rng](
|
||||||
const EnrichedCity& enriched_city) -> std::optional<BreweryRecord> {
|
const EnrichedCity& enriched_city) -> std::optional<BreweryRecord> {
|
||||||
try {
|
try {
|
||||||
const BreweryResult brewery =
|
const BreweryResult brewery =
|
||||||
generator_->GenerateBrewery(enriched_city);
|
generator_->GenerateBrewery(enriched_city);
|
||||||
const std::string postal_code =
|
const std::string postal_code =
|
||||||
postal_code_service_->GeneratePostalCode(enriched_city.location);
|
postal_code_service_->GeneratePostalCode(enriched_city.location);
|
||||||
|
|
||||||
|
constexpr int kMaxDistanceFromCentreKm = 5;
|
||||||
|
const Coords city_centre{.longitude = enriched_city.location.longitude,
|
||||||
|
.latitude = enriched_city.location.latitude};
|
||||||
|
const Coords brewery_coords = GetRandomCoordsWithinRange(
|
||||||
|
city_centre, kMaxDistanceFromCentreKm, rng);
|
||||||
|
|
||||||
return BreweryRecord{
|
return BreweryRecord{
|
||||||
.address = BreweryAddress{.city = enriched_city.location,
|
.address = BreweryAddress{.city = enriched_city.location,
|
||||||
.postal_code = postal_code},
|
.postal_code = postal_code,
|
||||||
|
.longitude = brewery_coords.longitude,
|
||||||
|
.latitude = brewery_coords.latitude},
|
||||||
.brewery = brewery};
|
.brewery = brewery};
|
||||||
} catch (const std::exception& e) {
|
} catch (const std::exception& e) {
|
||||||
++skipped_count;
|
++skipped_count;
|
||||||
@@ -91,4 +144,4 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
|
|||||||
"brewery/breweries to SQLite",
|
"brewery/breweries to SQLite",
|
||||||
export_failed_count)});
|
export_failed_count)});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -17,17 +17,19 @@ void BiergartenPipelineOrchestrator::LogResults() const {
|
|||||||
for (const auto& [address, brewery] : generated_breweries_) {
|
for (const auto& [address, brewery] : generated_breweries_) {
|
||||||
const City& city = address.city;
|
const City& city = address.city;
|
||||||
brewery_output.push_back(boost::json::object{
|
brewery_output.push_back(boost::json::object{
|
||||||
{"name_en", brewery.name_en},
|
{"name_en", brewery.name_en},
|
||||||
{"description_en", brewery.description_en},
|
{"description_en", brewery.description_en},
|
||||||
{"name_local", brewery.name_local},
|
{"name_local", brewery.name_local},
|
||||||
{"description_local", brewery.description_local},
|
{"description_local", brewery.description_local},
|
||||||
{"address", boost::json::object{
|
{"address", boost::json::object{
|
||||||
{"city", city.city},
|
{"city", city.city},
|
||||||
{"country", city.country},
|
{"country", city.country},
|
||||||
{"state_province", city.state_province},
|
{"state_province", city.state_province},
|
||||||
{"iso3166_2", city.iso3166_2},
|
{"iso3166_2", city.iso3166_2},
|
||||||
{"postal_code", address.postal_code},
|
{"postal_code", address.postal_code},
|
||||||
}}});
|
{"longitude", address.longitude},
|
||||||
|
{"latitude", address.latitude},
|
||||||
|
}}});
|
||||||
}
|
}
|
||||||
|
|
||||||
std::ostringstream brewery_oss;
|
std::ostringstream brewery_oss;
|
||||||
@@ -40,14 +42,14 @@ void BiergartenPipelineOrchestrator::LogResults() const {
|
|||||||
|
|
||||||
for (const auto& generated_user : generated_users_) {
|
for (const auto& generated_user : generated_users_) {
|
||||||
user_output.push_back(boost::json::object{
|
user_output.push_back(boost::json::object{
|
||||||
{"first_name", generated_user.user.first_name},
|
{"first_name", generated_user.user.first_name},
|
||||||
{"last_name", generated_user.user.last_name},
|
{"last_name", generated_user.user.last_name},
|
||||||
{"gender", generated_user.user.gender},
|
{"gender", generated_user.user.gender},
|
||||||
{"username", generated_user.user.username},
|
{"username", generated_user.user.username},
|
||||||
{"bio", generated_user.user.bio},
|
{"bio", generated_user.user.bio},
|
||||||
{"activity_weight", generated_user.user.activity_weight},
|
{"activity_weight", generated_user.user.activity_weight},
|
||||||
{"email", generated_user.email},
|
{"email", generated_user.email},
|
||||||
{"date_of_birth", generated_user.date_of_birth},
|
{"date_of_birth", generated_user.date_of_birth},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -56,4 +58,4 @@ void BiergartenPipelineOrchestrator::LogResults() const {
|
|||||||
logger_->Log({.level = LogLevel::Info,
|
logger_->Log({.level = LogLevel::Info,
|
||||||
.phase = PipelinePhase::Teardown,
|
.phase = PipelinePhase::Teardown,
|
||||||
.message = user_oss.str()});
|
.message = user_oss.str()});
|
||||||
}
|
}
|
||||||
@@ -39,11 +39,10 @@ std::vector<City> BiergartenPipelineOrchestrator::QueryLocations() {
|
|||||||
|
|
||||||
logger_->Log({.level = LogLevel::Info,
|
logger_->Log({.level = LogLevel::Info,
|
||||||
.phase = PipelinePhase::Startup,
|
.phase = PipelinePhase::Startup,
|
||||||
.message = std::format(" Locations available: {}",
|
.message = std::format(
|
||||||
all_locations.size())});
|
" Locations available: {}\n Sampled Locations: {}",
|
||||||
logger_->Log({.level = LogLevel::Info,
|
all_locations.size(), sampled_locations.size())});
|
||||||
.phase = PipelinePhase::Startup,
|
|
||||||
.message = std::format(" Sampled locations: {}",
|
|
||||||
sampled_locations.size())});
|
|
||||||
return sampled_locations;
|
return sampled_locations;
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,236 @@
|
|||||||
|
/**
|
||||||
|
* @file data_generation/generation_json_validation.cc
|
||||||
|
* @brief Generator-agnostic prompt-context preparation and structured JSON
|
||||||
|
* validation, shared across all DataGenerator implementations.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include "data_generation/generation_json_validation.h"
|
||||||
|
|
||||||
|
#include <algorithm>
|
||||||
|
#include <array>
|
||||||
|
#include <cctype>
|
||||||
|
#include <format>
|
||||||
|
#include <span>
|
||||||
|
#include <string>
|
||||||
|
#include <string_view>
|
||||||
|
|
||||||
|
#include <boost/json.hpp>
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
/**
|
||||||
|
* String trimming: removes leading and trailing whitespace
|
||||||
|
*/
|
||||||
|
std::string Trim(std::string_view value) {
|
||||||
|
constexpr std::string_view whitespace = " \t\n\r\f\v";
|
||||||
|
const size_t first_index = value.find_first_not_of(whitespace);
|
||||||
|
if (first_index == std::string_view::npos) {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
const size_t last_index = value.find_last_not_of(whitespace);
|
||||||
|
return std::string(value.substr(first_index, last_index - first_index + 1));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Normalize whitespace: collapses multiple spaces/tabs/newlines into single
|
||||||
|
* spaces
|
||||||
|
*/
|
||||||
|
std::string CondenseWhitespace(std::string_view text) {
|
||||||
|
std::string out;
|
||||||
|
out.reserve(text.size());
|
||||||
|
|
||||||
|
bool pending_space = false;
|
||||||
|
for (const char chr : text) {
|
||||||
|
if (std::isspace(static_cast<unsigned char>(chr)) != 0) {
|
||||||
|
if (!out.empty()) {
|
||||||
|
pending_space = true;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (pending_space) {
|
||||||
|
out.push_back(' ');
|
||||||
|
pending_space = false;
|
||||||
|
}
|
||||||
|
out.push_back(chr);
|
||||||
|
}
|
||||||
|
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Guard against truncating in the first half of the string.
|
||||||
|
// This preserves the critical opening content and avoids cutting critical
|
||||||
|
// context words early in the region description.
|
||||||
|
constexpr size_t kTruncationGuardDivisor = 2;
|
||||||
|
|
||||||
|
bool ReadRequiredTrimmedStringField(const boost::json::object& obj,
|
||||||
|
std::string_view key, std::string& out,
|
||||||
|
std::string* error_out) {
|
||||||
|
const boost::json::value* field = obj.if_contains(key);
|
||||||
|
if (field == nullptr || !field->is_string()) {
|
||||||
|
if (error_out != nullptr) {
|
||||||
|
*error_out =
|
||||||
|
std::format("Missing or invalid string field: {}", key);
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const auto& string_value = field->as_string();
|
||||||
|
out = Trim(std::string_view(string_value.data(), string_value.size()));
|
||||||
|
if (out.empty() && error_out != nullptr) {
|
||||||
|
*error_out = std::format("Field must not be empty: {}", key);
|
||||||
|
}
|
||||||
|
return !out.empty();
|
||||||
|
}
|
||||||
|
|
||||||
|
bool HasSchemaPlaceholder(std::span<std::string* const> values) {
|
||||||
|
for (const std::string* value : values) {
|
||||||
|
std::string lowered = *value;
|
||||||
|
std::ranges::transform(
|
||||||
|
lowered, lowered.begin(), [](const unsigned char character) {
|
||||||
|
return static_cast<char>(std::tolower(character));
|
||||||
|
});
|
||||||
|
|
||||||
|
if (lowered == "string") {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Truncate region context to fit within max length while preserving word
|
||||||
|
* boundaries
|
||||||
|
*/
|
||||||
|
std::string PrepareRegionContext(std::string_view region_context,
|
||||||
|
const size_t max_chars) {
|
||||||
|
std::string normalized = CondenseWhitespace(region_context);
|
||||||
|
if (normalized.size() <= max_chars) {
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
normalized.resize(max_chars);
|
||||||
|
const size_t last_space = normalized.find_last_of(' ');
|
||||||
|
if (last_space != std::string::npos &&
|
||||||
|
last_space > max_chars / kTruncationGuardDivisor) {
|
||||||
|
normalized.resize(last_space);
|
||||||
|
}
|
||||||
|
|
||||||
|
normalized += "...";
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::optional<std::string> ValidateBreweryJson(const std::string& raw,
|
||||||
|
BreweryResult& brewery_out) {
|
||||||
|
boost::system::error_code error_code;
|
||||||
|
const std::string_view raw_view(raw);
|
||||||
|
const size_t opening_brace = raw_view.find('{');
|
||||||
|
if (opening_brace == std::string_view::npos) {
|
||||||
|
return "JSON parse error: missing opening brace '{'";
|
||||||
|
}
|
||||||
|
|
||||||
|
const std::string_view json_payload = raw_view.substr(opening_brace);
|
||||||
|
boost::json::value json_value = boost::json::parse(json_payload, error_code);
|
||||||
|
if (error_code) {
|
||||||
|
return "JSON parse error: " + error_code.message();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!json_value.is_object()) {
|
||||||
|
return "JSON root must be an object";
|
||||||
|
}
|
||||||
|
|
||||||
|
const auto& obj = json_value.get_object();
|
||||||
|
if (obj.size() != 4) {
|
||||||
|
return "JSON object must contain exactly four keys";
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string validation_error;
|
||||||
|
if (!ReadRequiredTrimmedStringField(obj, "name_en", brewery_out.name_en,
|
||||||
|
&validation_error)) {
|
||||||
|
return validation_error;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!ReadRequiredTrimmedStringField(obj, "description_en",
|
||||||
|
brewery_out.description_en,
|
||||||
|
&validation_error)) {
|
||||||
|
return validation_error;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!ReadRequiredTrimmedStringField(
|
||||||
|
obj, "name_local", brewery_out.name_local, &validation_error)) {
|
||||||
|
return validation_error;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!ReadRequiredTrimmedStringField(obj, "description_local",
|
||||||
|
brewery_out.description_local,
|
||||||
|
&validation_error)) {
|
||||||
|
return validation_error;
|
||||||
|
}
|
||||||
|
|
||||||
|
const std::array schema_placeholders = {
|
||||||
|
&brewery_out.name_en, &brewery_out.description_en,
|
||||||
|
&brewery_out.name_local, &brewery_out.description_local};
|
||||||
|
if (HasSchemaPlaceholder(schema_placeholders)) {
|
||||||
|
return "JSON appears to be a schema placeholder, not content";
|
||||||
|
}
|
||||||
|
|
||||||
|
return std::nullopt;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::optional<std::string> ValidateUserJson(const std::string& raw,
|
||||||
|
UserResult& user_out) {
|
||||||
|
boost::system::error_code error_code;
|
||||||
|
const std::string_view raw_view(raw);
|
||||||
|
const size_t opening_brace = raw_view.find('{');
|
||||||
|
if (opening_brace == std::string_view::npos) {
|
||||||
|
return "JSON parse error: missing opening brace '{'";
|
||||||
|
}
|
||||||
|
|
||||||
|
const std::string_view json_payload = raw_view.substr(opening_brace);
|
||||||
|
boost::json::value json_value = boost::json::parse(json_payload, error_code);
|
||||||
|
if (error_code) {
|
||||||
|
return "JSON parse error: " + error_code.message();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!json_value.is_object()) {
|
||||||
|
return "JSON root must be an object";
|
||||||
|
}
|
||||||
|
|
||||||
|
const auto& obj = json_value.get_object();
|
||||||
|
if (obj.size() != 3) {
|
||||||
|
return "JSON object must contain exactly three keys";
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string validation_error;
|
||||||
|
if (!ReadRequiredTrimmedStringField(obj, "username", user_out.username,
|
||||||
|
&validation_error)) {
|
||||||
|
return validation_error;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!ReadRequiredTrimmedStringField(obj, "bio", user_out.bio,
|
||||||
|
&validation_error)) {
|
||||||
|
return validation_error;
|
||||||
|
}
|
||||||
|
|
||||||
|
const boost::json::value* activity_weight_field =
|
||||||
|
obj.if_contains("activity_weight");
|
||||||
|
if (activity_weight_field == nullptr ||
|
||||||
|
!activity_weight_field->is_number()) {
|
||||||
|
return "Missing or invalid numeric field: activity_weight";
|
||||||
|
}
|
||||||
|
|
||||||
|
const double activity_weight = activity_weight_field->to_number<double>();
|
||||||
|
if (activity_weight < 0.0 || activity_weight > 1.0) {
|
||||||
|
return "activity_weight must be between 0 and 1";
|
||||||
|
}
|
||||||
|
user_out.activity_weight = static_cast<float>(activity_weight);
|
||||||
|
|
||||||
|
const std::array schema_placeholders = {&user_out.username, &user_out.bio};
|
||||||
|
if (HasSchemaPlaceholder(schema_placeholders)) {
|
||||||
|
return "JSON appears to be a schema placeholder, not content";
|
||||||
|
}
|
||||||
|
|
||||||
|
return std::nullopt;
|
||||||
|
}
|
||||||
@@ -12,9 +12,9 @@
|
|||||||
#include <string_view>
|
#include <string_view>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
|
#include "data_generation/generation_json_validation.h"
|
||||||
#include "data_generation/json_grammars.h"
|
#include "data_generation/json_grammars.h"
|
||||||
#include "data_generation/llama_generator.h"
|
#include "data_generation/llama_generator.h"
|
||||||
#include "data_generation/llama_generator_helpers.h"
|
|
||||||
|
|
||||||
static std::string FormatLocalLanguageCodes(
|
static std::string FormatLocalLanguageCodes(
|
||||||
const std::vector<std::string>& codes) {
|
const std::vector<std::string>& codes) {
|
||||||
|
|||||||
@@ -10,9 +10,9 @@
|
|||||||
#include <string>
|
#include <string>
|
||||||
#include <string_view>
|
#include <string_view>
|
||||||
|
|
||||||
|
#include "data_generation/generation_json_validation.h"
|
||||||
#include "data_generation/json_grammars.h"
|
#include "data_generation/json_grammars.h"
|
||||||
#include "data_generation/llama_generator.h"
|
#include "data_generation/llama_generator.h"
|
||||||
#include "data_generation/llama_generator_helpers.h"
|
|
||||||
|
|
||||||
static constexpr int kUserInitialMaxTokens = 1200;
|
static constexpr int kUserInitialMaxTokens = 1200;
|
||||||
|
|
||||||
|
|||||||
@@ -1,123 +1,17 @@
|
|||||||
/**
|
/**
|
||||||
* @file data_generation/llama/helpers.cc
|
* @file data_generation/llama/helpers.cc
|
||||||
* @brief Provides prompt formatting, whitespace normalization, response
|
* @brief Token decoding helper for LlamaGenerator modules.
|
||||||
* parsing, token decoding, and JSON validation helpers for Llama modules.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include <llama.h>
|
#include <llama.h>
|
||||||
|
|
||||||
#include <algorithm>
|
|
||||||
#include <array>
|
#include <array>
|
||||||
#include <boost/json.hpp>
|
|
||||||
#include <cctype>
|
|
||||||
#include <optional>
|
|
||||||
#include <span>
|
|
||||||
#include <stdexcept>
|
#include <stdexcept>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <string_view>
|
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "data_generation/llama_generator_helpers.h"
|
#include "data_generation/llama_generator_helpers.h"
|
||||||
|
|
||||||
namespace {
|
|
||||||
/**
|
|
||||||
* String trimming: removes leading and trailing whitespace
|
|
||||||
*/
|
|
||||||
std::string Trim(std::string_view value) {
|
|
||||||
constexpr std::string_view whitespace = " \t\n\r\f\v";
|
|
||||||
const size_t first_index = value.find_first_not_of(whitespace);
|
|
||||||
if (first_index == std::string_view::npos) {
|
|
||||||
return {};
|
|
||||||
}
|
|
||||||
|
|
||||||
const size_t last_index = value.find_last_not_of(whitespace);
|
|
||||||
return std::string(value.substr(first_index, last_index - first_index + 1));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Normalize whitespace: collapses multiple spaces/tabs/newlines into single
|
|
||||||
* spaces
|
|
||||||
*/
|
|
||||||
std::string CondenseWhitespace(std::string_view text) {
|
|
||||||
std::string out;
|
|
||||||
out.reserve(text.size());
|
|
||||||
|
|
||||||
bool pending_space = false;
|
|
||||||
for (const char chr : text) {
|
|
||||||
if (std::isspace(static_cast<unsigned char>(chr)) != 0) {
|
|
||||||
if (!out.empty()) {
|
|
||||||
pending_space = true;
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (pending_space) {
|
|
||||||
out.push_back(' ');
|
|
||||||
pending_space = false;
|
|
||||||
}
|
|
||||||
out.push_back(chr);
|
|
||||||
}
|
|
||||||
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Guard against truncating in the first half of the string.
|
|
||||||
// This preserves the critical opening content and avoids cutting critical
|
|
||||||
// context words early in the region description.
|
|
||||||
constexpr size_t kTruncationGuardDivisor = 2;
|
|
||||||
|
|
||||||
bool ReadRequiredTrimmedStringField(const boost::json::object& obj,
|
|
||||||
std::string_view key, std::string& out,
|
|
||||||
std::string* error_out) {
|
|
||||||
const boost::json::value* field = obj.if_contains(key);
|
|
||||||
if (field == nullptr || !field->is_string()) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
const auto& string_value = field->as_string();
|
|
||||||
out = Trim(std::string_view(string_value.data(), string_value.size()));
|
|
||||||
return !out.empty();
|
|
||||||
}
|
|
||||||
|
|
||||||
bool HasSchemaPlaceholder(std::span<std::string* const> values) {
|
|
||||||
for (const std::string* value : values) {
|
|
||||||
std::string lowered = *value;
|
|
||||||
std::ranges::transform(
|
|
||||||
lowered, lowered.begin(), [](const unsigned char character) {
|
|
||||||
return static_cast<char>(std::tolower(character));
|
|
||||||
});
|
|
||||||
|
|
||||||
if (lowered == "string") {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
} // namespace
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Truncate region context to fit within max length while preserving word
|
|
||||||
* boundaries
|
|
||||||
*/
|
|
||||||
std::string PrepareRegionContext(std::string_view region_context,
|
|
||||||
const size_t max_chars) {
|
|
||||||
std::string normalized = CondenseWhitespace(region_context);
|
|
||||||
if (normalized.size() <= max_chars) {
|
|
||||||
return normalized;
|
|
||||||
}
|
|
||||||
|
|
||||||
normalized.resize(max_chars);
|
|
||||||
const size_t last_space = normalized.find_last_of(' ');
|
|
||||||
if (last_space != std::string::npos &&
|
|
||||||
last_space > max_chars / kTruncationGuardDivisor) {
|
|
||||||
normalized.resize(last_space);
|
|
||||||
}
|
|
||||||
|
|
||||||
normalized += "...";
|
|
||||||
return normalized;
|
|
||||||
}
|
|
||||||
|
|
||||||
void AppendTokenPiece(const llama_vocab* vocab, llama_token token,
|
void AppendTokenPiece(const llama_vocab* vocab, llama_token token,
|
||||||
std::string& output) {
|
std::string& output) {
|
||||||
constexpr size_t initial_buffer_size = 256;
|
constexpr size_t initial_buffer_size = 256;
|
||||||
@@ -153,116 +47,3 @@ void AppendTokenPiece(const llama_vocab* vocab, llama_token token,
|
|||||||
throw std::runtime_error(
|
throw std::runtime_error(
|
||||||
"LlamaGenerator: failed to decode sampled token piece");
|
"LlamaGenerator: failed to decode sampled token piece");
|
||||||
}
|
}
|
||||||
|
|
||||||
std::optional<std::string> ValidateBreweryJson(const std::string& raw,
|
|
||||||
BreweryResult& brewery_out) {
|
|
||||||
boost::system::error_code error_code;
|
|
||||||
const std::string_view raw_view(raw);
|
|
||||||
const size_t opening_brace = raw_view.find('{');
|
|
||||||
if (opening_brace == std::string_view::npos) {
|
|
||||||
return "JSON parse error: missing opening brace '{'";
|
|
||||||
}
|
|
||||||
|
|
||||||
const std::string_view json_payload = raw_view.substr(opening_brace);
|
|
||||||
boost::json::value json_value = boost::json::parse(json_payload, error_code);
|
|
||||||
if (error_code) {
|
|
||||||
return "JSON parse error: " + error_code.message();
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!json_value.is_object()) {
|
|
||||||
return "JSON root must be an object";
|
|
||||||
}
|
|
||||||
|
|
||||||
const auto& obj = json_value.get_object();
|
|
||||||
if (obj.size() != 4) {
|
|
||||||
return "JSON object must contain exactly four keys";
|
|
||||||
}
|
|
||||||
|
|
||||||
std::string validation_error;
|
|
||||||
if (!ReadRequiredTrimmedStringField(obj, "name_en", brewery_out.name_en,
|
|
||||||
&validation_error)) {
|
|
||||||
return validation_error;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!ReadRequiredTrimmedStringField(obj, "description_en",
|
|
||||||
brewery_out.description_en,
|
|
||||||
&validation_error)) {
|
|
||||||
return validation_error;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!ReadRequiredTrimmedStringField(
|
|
||||||
obj, "name_local", brewery_out.name_local, &validation_error)) {
|
|
||||||
return validation_error;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!ReadRequiredTrimmedStringField(obj, "description_local",
|
|
||||||
brewery_out.description_local,
|
|
||||||
&validation_error)) {
|
|
||||||
return validation_error;
|
|
||||||
}
|
|
||||||
|
|
||||||
const std::array schema_placeholders = {
|
|
||||||
&brewery_out.name_en, &brewery_out.description_en,
|
|
||||||
&brewery_out.name_local, &brewery_out.description_local};
|
|
||||||
if (HasSchemaPlaceholder(schema_placeholders)) {
|
|
||||||
return "JSON appears to be a schema placeholder, not content";
|
|
||||||
}
|
|
||||||
|
|
||||||
return std::nullopt;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::optional<std::string> ValidateUserJson(const std::string& raw,
|
|
||||||
UserResult& user_out) {
|
|
||||||
boost::system::error_code error_code;
|
|
||||||
const std::string_view raw_view(raw);
|
|
||||||
const size_t opening_brace = raw_view.find('{');
|
|
||||||
if (opening_brace == std::string_view::npos) {
|
|
||||||
return "JSON parse error: missing opening brace '{'";
|
|
||||||
}
|
|
||||||
|
|
||||||
const std::string_view json_payload = raw_view.substr(opening_brace);
|
|
||||||
boost::json::value json_value = boost::json::parse(json_payload, error_code);
|
|
||||||
if (error_code) {
|
|
||||||
return "JSON parse error: " + error_code.message();
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!json_value.is_object()) {
|
|
||||||
return "JSON root must be an object";
|
|
||||||
}
|
|
||||||
|
|
||||||
const auto& obj = json_value.get_object();
|
|
||||||
if (obj.size() != 3) {
|
|
||||||
return "JSON object must contain exactly three keys";
|
|
||||||
}
|
|
||||||
|
|
||||||
std::string validation_error;
|
|
||||||
if (!ReadRequiredTrimmedStringField(obj, "username", user_out.username,
|
|
||||||
&validation_error)) {
|
|
||||||
return validation_error;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!ReadRequiredTrimmedStringField(obj, "bio", user_out.bio,
|
|
||||||
&validation_error)) {
|
|
||||||
return validation_error;
|
|
||||||
}
|
|
||||||
|
|
||||||
const boost::json::value* activity_weight_field =
|
|
||||||
obj.if_contains("activity_weight");
|
|
||||||
if (activity_weight_field == nullptr ||
|
|
||||||
!activity_weight_field->is_number()) {
|
|
||||||
return "Missing or invalid numeric field: activity_weight";
|
|
||||||
}
|
|
||||||
|
|
||||||
const double activity_weight = activity_weight_field->to_number<double>();
|
|
||||||
if (activity_weight < 0.0 || activity_weight > 1.0) {
|
|
||||||
return "activity_weight must be between 0 and 1";
|
|
||||||
}
|
|
||||||
user_out.activity_weight = static_cast<float>(activity_weight);
|
|
||||||
|
|
||||||
const std::array schema_placeholders = {&user_out.username, &user_out.bio};
|
|
||||||
if (HasSchemaPlaceholder(schema_placeholders)) {
|
|
||||||
return "JSON appears to be a schema placeholder, not content";
|
|
||||||
}
|
|
||||||
|
|
||||||
return std::nullopt;
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,182 @@
|
|||||||
|
/**
|
||||||
|
* @file data_generation/openai/call_chat_completions_api.cc
|
||||||
|
* @brief OpenAIGenerator::CallChatCompletionsApi() -- builds and sends a
|
||||||
|
* single OpenAI Chat Completions request with a Structured Outputs JSON
|
||||||
|
* schema, and extracts the resulting message content.
|
||||||
|
*
|
||||||
|
* API docs last accessed 2026-08-23.
|
||||||
|
* Docs: https://platform.openai.com/docs/api-reference/chat/create
|
||||||
|
* Structured Outputs: https://platform.openai.com/docs/guides/structured-outputs
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <format>
|
||||||
|
#include <stdexcept>
|
||||||
|
#include <string>
|
||||||
|
#include <string_view>
|
||||||
|
#include <utility>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include <boost/json.hpp>
|
||||||
|
|
||||||
|
#include "data_generation/openai_generator.h"
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
// https://platform.openai.com/docs/api-reference/chat/create
|
||||||
|
constexpr std::string_view kChatCompletionsUrl =
|
||||||
|
"https://api.openai.com/v1/chat/completions";
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Sends a single Chat Completions request constrained by a JSON
|
||||||
|
* Schema and returns the assistant message's text content.
|
||||||
|
*
|
||||||
|
* Request/response fields follow the Chat Completions API reference:
|
||||||
|
* https://platform.openai.com/docs/api-reference/chat/create
|
||||||
|
* Structured Outputs (response_format, strict mode, refusal field):
|
||||||
|
* https://platform.openai.com/docs/guides/structured-outputs
|
||||||
|
*
|
||||||
|
* @param system_prompt Text sent as the "system" role message.
|
||||||
|
* @param user_prompt Text sent as the "user" role message.
|
||||||
|
* @param json_schema JSON Schema the response must conform to.
|
||||||
|
* @param schema_name Identifier for response_format.json_schema.name.
|
||||||
|
* @param max_tokens Value sent as "max_completion_tokens".
|
||||||
|
* @return The raw JSON string from choices[0].message.content.
|
||||||
|
* @throws std::runtime_error on invalid schema, API error, malformed
|
||||||
|
* response, model refusal, or missing content.
|
||||||
|
*/
|
||||||
|
std::string OpenAIGenerator::CallChatCompletionsApi(
|
||||||
|
const std::string& system_prompt, const std::string& user_prompt,
|
||||||
|
std::string_view json_schema, std::string_view schema_name,
|
||||||
|
int max_tokens) {
|
||||||
|
|
||||||
|
boost::system::error_code schema_error;
|
||||||
|
boost::json::value schema_value =
|
||||||
|
boost::json::parse(json_schema, schema_error);
|
||||||
|
|
||||||
|
bool schema_is_invalid = static_cast<bool>(schema_error);
|
||||||
|
if (schema_is_invalid) {
|
||||||
|
throw std::runtime_error(
|
||||||
|
"OpenAIGenerator: invalid embedded JSON schema: " +
|
||||||
|
schema_error.message());
|
||||||
|
}
|
||||||
|
|
||||||
|
// response_format: https://platform.openai.com/docs/guides/structured-outputs
|
||||||
|
boost::json::object json_schema_wrapper;
|
||||||
|
json_schema_wrapper["name"] = std::string(schema_name);
|
||||||
|
json_schema_wrapper["strict"] = true;
|
||||||
|
json_schema_wrapper["schema"] = std::move(schema_value);
|
||||||
|
|
||||||
|
boost::json::object response_format;
|
||||||
|
response_format["type"] = "json_schema";
|
||||||
|
response_format["json_schema"] = std::move(json_schema_wrapper);
|
||||||
|
|
||||||
|
boost::json::object system_message;
|
||||||
|
system_message["role"] = "system";
|
||||||
|
system_message["content"] = system_prompt;
|
||||||
|
|
||||||
|
boost::json::object user_message;
|
||||||
|
user_message["role"] = "user";
|
||||||
|
user_message["content"] = user_prompt;
|
||||||
|
|
||||||
|
boost::json::array messages;
|
||||||
|
messages.push_back(std::move(system_message));
|
||||||
|
messages.push_back(std::move(user_message));
|
||||||
|
|
||||||
|
boost::json::object request;
|
||||||
|
request["model"] = model_;
|
||||||
|
request["max_completion_tokens"] = max_tokens;
|
||||||
|
request["messages"] = std::move(messages);
|
||||||
|
request["response_format"] = std::move(response_format);
|
||||||
|
|
||||||
|
const std::string body = boost::json::serialize(request);
|
||||||
|
|
||||||
|
const std::vector<std::pair<std::string, std::string>> headers = {
|
||||||
|
{"Authorization", std::format("Bearer {}", api_key_)},
|
||||||
|
};
|
||||||
|
|
||||||
|
const std::string response_body =
|
||||||
|
web_client_->Post(std::string(kChatCompletionsUrl), body, headers);
|
||||||
|
|
||||||
|
boost::system::error_code response_error;
|
||||||
|
boost::json::value response_value =
|
||||||
|
boost::json::parse(response_body, response_error);
|
||||||
|
|
||||||
|
bool response_body_is_invalid = static_cast<bool>(response_error);
|
||||||
|
if (response_body_is_invalid) {
|
||||||
|
throw std::runtime_error(
|
||||||
|
"OpenAIGenerator: failed to parse API response: " +
|
||||||
|
response_error.message());
|
||||||
|
}
|
||||||
|
|
||||||
|
bool response_root_is_not_object = !response_value.is_object();
|
||||||
|
if (response_root_is_not_object) {
|
||||||
|
throw std::runtime_error(
|
||||||
|
"OpenAIGenerator: API response root is not an object");
|
||||||
|
}
|
||||||
|
const auto& response_obj = response_value.get_object();
|
||||||
|
|
||||||
|
// Error object: https://platform.openai.com/docs/api-reference/chat/create
|
||||||
|
const boost::json::value* error_field = response_obj.if_contains("error");
|
||||||
|
|
||||||
|
bool top_level_error_occurred =
|
||||||
|
error_field != nullptr && error_field->is_object();
|
||||||
|
if (top_level_error_occurred) {
|
||||||
|
std::string error_message = "unknown error";
|
||||||
|
if (const boost::json::value* message_field =
|
||||||
|
error_field->get_object().if_contains("message");
|
||||||
|
message_field != nullptr && message_field->is_string()) {
|
||||||
|
error_message = std::string(message_field->as_string());
|
||||||
|
}
|
||||||
|
throw std::runtime_error(
|
||||||
|
std::format("OpenAIGenerator: API request failed: {}", error_message));
|
||||||
|
}
|
||||||
|
|
||||||
|
const boost::json::value* choices_field = response_obj.
|
||||||
|
if_contains("choices");
|
||||||
|
bool choices_field_missing_or_invalid =
|
||||||
|
choices_field == nullptr || !choices_field->is_array() ||
|
||||||
|
choices_field->get_array().empty();
|
||||||
|
|
||||||
|
if (choices_field_missing_or_invalid) {
|
||||||
|
throw std::runtime_error(
|
||||||
|
"OpenAIGenerator: API response missing choices array");
|
||||||
|
}
|
||||||
|
|
||||||
|
const boost::json::value& first_choice = choices_field->get_array().front();
|
||||||
|
bool first_choice_is_not_object = !first_choice.is_object();
|
||||||
|
if (first_choice_is_not_object) {
|
||||||
|
throw std::runtime_error(
|
||||||
|
"OpenAIGenerator: API response choice is not an object");
|
||||||
|
}
|
||||||
|
const auto& choice_obj = first_choice.get_object();
|
||||||
|
|
||||||
|
const boost::json::value* message_field = choice_obj.if_contains("message");
|
||||||
|
bool message_field_missing_or_invalid =
|
||||||
|
message_field == nullptr || !message_field->is_object();
|
||||||
|
if (message_field_missing_or_invalid) {
|
||||||
|
throw std::runtime_error(
|
||||||
|
"OpenAIGenerator: API response choice missing message");
|
||||||
|
}
|
||||||
|
const auto& message_obj = message_field->get_object();
|
||||||
|
|
||||||
|
// Refusal field: https://platform.openai.com/docs/guides/structured-outputs
|
||||||
|
const boost::json::value* refusal_field = message_obj.if_contains("refusal");
|
||||||
|
bool model_refused_request =
|
||||||
|
refusal_field != nullptr && refusal_field->is_string() &&
|
||||||
|
!refusal_field->as_string().empty();
|
||||||
|
if (model_refused_request) {
|
||||||
|
throw std::runtime_error(std::format(
|
||||||
|
"OpenAIGenerator: request refused by safety policies: {}",
|
||||||
|
std::string(refusal_field->as_string())));
|
||||||
|
}
|
||||||
|
|
||||||
|
const boost::json::value* content_field = message_obj.if_contains("content");
|
||||||
|
bool content_field_missing_or_invalid =
|
||||||
|
content_field == nullptr || !content_field->is_string();
|
||||||
|
if (content_field_missing_or_invalid) {
|
||||||
|
throw std::runtime_error(
|
||||||
|
"OpenAIGenerator: API response message had no text content");
|
||||||
|
}
|
||||||
|
|
||||||
|
return std::string(content_field->as_string());
|
||||||
|
}
|
||||||
132
tooling/pipeline/src/data_generation/openai/generate_brewery.cc
Normal file
132
tooling/pipeline/src/data_generation/openai/generate_brewery.cc
Normal file
@@ -0,0 +1,132 @@
|
|||||||
|
/**
|
||||||
|
* @file data_generation/openai/generate_brewery.cc
|
||||||
|
* @brief Builds brewery prompts with regional context, calls the OpenAI
|
||||||
|
* Chat Completions API, and validates structured JSON output for brewery
|
||||||
|
* records.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <format>
|
||||||
|
#include <optional>
|
||||||
|
#include <stdexcept>
|
||||||
|
#include <string>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include "data_generation/generation_json_validation.h"
|
||||||
|
#include "data_generation/openai_generator.h"
|
||||||
|
#include "data_generation/openai_json_schemas.h"
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
|
||||||
|
std::string FormatLocalLanguageCodes(
|
||||||
|
const std::vector<std::string>& codes) {
|
||||||
|
if (codes.empty()) {
|
||||||
|
return "Not provided";
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string formatted;
|
||||||
|
for (const std::string& code : codes) {
|
||||||
|
if (!formatted.empty()) {
|
||||||
|
formatted += ", ";
|
||||||
|
}
|
||||||
|
formatted += code;
|
||||||
|
}
|
||||||
|
|
||||||
|
return formatted;
|
||||||
|
}
|
||||||
|
|
||||||
|
constexpr int kBreweryMaxTokens = 2800;
|
||||||
|
|
||||||
|
// Structured outputs already guarantee schema-valid JSON, so retries here
|
||||||
|
// are only for transient failures or the model returning placeholder text
|
||||||
|
// -- not for malformed JSON shape, unlike LlamaGenerator's grammar-assisted
|
||||||
|
// retry loop.
|
||||||
|
constexpr int kMaxAttempts = 2;
|
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
BreweryResult OpenAIGenerator::GenerateBrewery(
|
||||||
|
const EnrichedCity& enriched_city) {
|
||||||
|
const City& location = enriched_city.location;
|
||||||
|
const std::string& region_context = enriched_city.region_context;
|
||||||
|
const std::string safe_region_context = PrepareRegionContext(region_context);
|
||||||
|
|
||||||
|
const std::string local_language_codes =
|
||||||
|
FormatLocalLanguageCodes(location.local_languages);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Load brewery system prompt via the injected prompt directory. The
|
||||||
|
* same file LlamaGenerator uses -- the prompt content is backend
|
||||||
|
* agnostic.
|
||||||
|
*/
|
||||||
|
const std::string system_prompt =
|
||||||
|
prompt_directory_->Load("BREWERY_GENERATION");
|
||||||
|
|
||||||
|
std::string user_prompt = std::format(
|
||||||
|
"## CITY:\n{}\n\n## COUNTRY:\n{}\n\n## LOCAL LANGUAGE CODES:\n{}\n\n## "
|
||||||
|
"CONTEXT:\n{}",
|
||||||
|
location.city, location.country, local_language_codes,
|
||||||
|
safe_region_context);
|
||||||
|
|
||||||
|
const std::string retry_location =
|
||||||
|
std::format("City: {}, {}\nLocal language codes: {}", location.city,
|
||||||
|
location.country, local_language_codes);
|
||||||
|
|
||||||
|
std::string raw;
|
||||||
|
std::string last_error;
|
||||||
|
|
||||||
|
for (int attempt = 0; attempt < kMaxAttempts; ++attempt) {
|
||||||
|
raw = CallChatCompletionsApi(system_prompt, user_prompt,
|
||||||
|
kBreweryJsonSchema, "brewery_result",
|
||||||
|
kBreweryMaxTokens);
|
||||||
|
if (logger_) {
|
||||||
|
logger_->Log({.level = LogLevel::Debug,
|
||||||
|
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
||||||
|
.message = std::format(
|
||||||
|
"OpenAIGenerator: raw output (attempt {}): {}",
|
||||||
|
attempt + 1, raw)});
|
||||||
|
}
|
||||||
|
|
||||||
|
BreweryResult brewery;
|
||||||
|
const std::optional<std::string> validation_error =
|
||||||
|
ValidateBreweryJson(raw, brewery);
|
||||||
|
|
||||||
|
if (!validation_error.has_value()) {
|
||||||
|
if (logger_) {
|
||||||
|
logger_->Log({.level = LogLevel::Info,
|
||||||
|
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
||||||
|
.message = std::format(
|
||||||
|
"OpenAIGenerator: successfully generated "
|
||||||
|
"brewery data on attempt {}",
|
||||||
|
attempt + 1)});
|
||||||
|
}
|
||||||
|
|
||||||
|
return brewery;
|
||||||
|
}
|
||||||
|
|
||||||
|
last_error = *validation_error;
|
||||||
|
if (logger_) {
|
||||||
|
logger_->Log(
|
||||||
|
{.level = LogLevel::Warn,
|
||||||
|
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
||||||
|
.message = std::format(
|
||||||
|
"OpenAIGenerator: malformed brewery JSON (attempt {}): {}",
|
||||||
|
attempt + 1, *validation_error)});
|
||||||
|
}
|
||||||
|
|
||||||
|
user_prompt = std::format(
|
||||||
|
"Your previous response was invalid. Error: {}\nReturn real "
|
||||||
|
"content matching the required schema -- do not return "
|
||||||
|
"placeholder values.\n\n{}",
|
||||||
|
*validation_error, retry_location);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (logger_) {
|
||||||
|
logger_->Log({.level = LogLevel::Error,
|
||||||
|
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
||||||
|
.message = std::format(
|
||||||
|
"OpenAIGenerator: malformed brewery "
|
||||||
|
"response after {} attempts: {}",
|
||||||
|
kMaxAttempts, last_error.empty() ? raw : last_error)});
|
||||||
|
}
|
||||||
|
throw std::runtime_error("OpenAIGenerator: malformed brewery response");
|
||||||
|
}
|
||||||
114
tooling/pipeline/src/data_generation/openai/generate_user.cc
Normal file
114
tooling/pipeline/src/data_generation/openai/generate_user.cc
Normal file
@@ -0,0 +1,114 @@
|
|||||||
|
/**
|
||||||
|
* @file data_generation/openai/generate_user.cc
|
||||||
|
* @brief Builds persona/name-grounded user prompts, calls the OpenAI Chat
|
||||||
|
* Completions API, and validates structured JSON output for user records.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <format>
|
||||||
|
#include <optional>
|
||||||
|
#include <stdexcept>
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
#include "data_generation/generation_json_validation.h"
|
||||||
|
#include "data_generation/openai_generator.h"
|
||||||
|
#include "data_generation/openai_json_schemas.h"
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
constexpr int kUserMaxTokens = 1200;
|
||||||
|
|
||||||
|
// Structured outputs already guarantee schema-valid JSON, so retries here
|
||||||
|
// are only for transient failures or placeholder text -- not JSON shape.
|
||||||
|
constexpr int kMaxAttempts = 2;
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
UserResult OpenAIGenerator::GenerateUser(const EnrichedCity& city,
|
||||||
|
const UserPersona& persona,
|
||||||
|
const Name& name) {
|
||||||
|
std::string style_affinities;
|
||||||
|
for (const std::string& style : persona.style_affinities) {
|
||||||
|
if (!style_affinities.empty()) {
|
||||||
|
style_affinities += ", ";
|
||||||
|
}
|
||||||
|
style_affinities += style;
|
||||||
|
}
|
||||||
|
|
||||||
|
const std::string system_prompt = prompt_directory_->Load("USER_GENERATION");
|
||||||
|
|
||||||
|
std::string user_prompt = std::format(
|
||||||
|
"## NAME:\n\n{} {}\n\n"
|
||||||
|
"## GENDER:\n\n{}\n\n"
|
||||||
|
"## CITY:\n\n{}\n\n"
|
||||||
|
"## COUNTRY:\n\n{}\n\n"
|
||||||
|
"## PERSONA:\n\n{}\n\n"
|
||||||
|
"## PERSONA DESCRIPTION:\n\n{}\n\n"
|
||||||
|
"## STYLE AFFINITIES:\n\n{}",
|
||||||
|
name.first_name, name.last_name, name.gender, city.location.city,
|
||||||
|
city.location.country, persona.name, persona.description,
|
||||||
|
style_affinities);
|
||||||
|
|
||||||
|
const std::string retry_context = std::format(
|
||||||
|
"Name: {} {}\nCity: {}, {}\nPersona: {}", name.first_name,
|
||||||
|
name.last_name, city.location.city, city.location.country,
|
||||||
|
persona.name);
|
||||||
|
|
||||||
|
std::string raw;
|
||||||
|
std::string last_error;
|
||||||
|
|
||||||
|
for (int attempt = 0; attempt < kMaxAttempts; ++attempt) {
|
||||||
|
raw = CallChatCompletionsApi(system_prompt, user_prompt, kUserJsonSchema,
|
||||||
|
"user_result", kUserMaxTokens);
|
||||||
|
if (logger_) {
|
||||||
|
logger_->Log({.level = LogLevel::Debug,
|
||||||
|
.phase = PipelinePhase::UserGeneration,
|
||||||
|
.message = std::format(
|
||||||
|
"OpenAIGenerator: raw output (attempt {}): {}",
|
||||||
|
attempt + 1, raw)});
|
||||||
|
}
|
||||||
|
|
||||||
|
UserResult user;
|
||||||
|
const std::optional<std::string> validation_error =
|
||||||
|
ValidateUserJson(raw, user);
|
||||||
|
|
||||||
|
if (!validation_error.has_value()) {
|
||||||
|
if (logger_) {
|
||||||
|
logger_->Log(
|
||||||
|
{.level = LogLevel::Info,
|
||||||
|
.phase = PipelinePhase::UserGeneration,
|
||||||
|
.message = std::format("OpenAIGenerator: successfully "
|
||||||
|
"generated user data on attempt {}",
|
||||||
|
attempt + 1)});
|
||||||
|
}
|
||||||
|
|
||||||
|
user.first_name = name.first_name;
|
||||||
|
user.last_name = name.last_name;
|
||||||
|
user.gender = name.gender;
|
||||||
|
return user;
|
||||||
|
}
|
||||||
|
|
||||||
|
last_error = *validation_error;
|
||||||
|
if (logger_) {
|
||||||
|
logger_->Log(
|
||||||
|
{.level = LogLevel::Warn,
|
||||||
|
.phase = PipelinePhase::UserGeneration,
|
||||||
|
.message = std::format(
|
||||||
|
"OpenAIGenerator: malformed user JSON (attempt {}): {}",
|
||||||
|
attempt + 1, *validation_error)});
|
||||||
|
}
|
||||||
|
|
||||||
|
user_prompt = std::format(
|
||||||
|
"Your previous response was invalid. Error: {}\nReturn real "
|
||||||
|
"content matching the required schema -- do not return "
|
||||||
|
"placeholder values.\n\n{}",
|
||||||
|
*validation_error, retry_context);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (logger_) {
|
||||||
|
logger_->Log({.level = LogLevel::Error,
|
||||||
|
.phase = PipelinePhase::UserGeneration,
|
||||||
|
.message = std::format(
|
||||||
|
"OpenAIGenerator: malformed user response "
|
||||||
|
"after {} attempts: {}",
|
||||||
|
kMaxAttempts, last_error.empty() ? raw : last_error)});
|
||||||
|
}
|
||||||
|
throw std::runtime_error("OpenAIGenerator: malformed user response");
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
/**
|
||||||
|
* @file data_generation/openai/openai_generator.cc
|
||||||
|
* @brief OpenAIGenerator constructor.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <memory>
|
||||||
|
#include <string>
|
||||||
|
#include <utility>
|
||||||
|
|
||||||
|
#include "data_generation/openai_generator.h"
|
||||||
|
|
||||||
|
OpenAIGenerator::OpenAIGenerator(
|
||||||
|
std::string api_key, std::string model, std::shared_ptr<ILogger> logger,
|
||||||
|
std::unique_ptr<IPromptDirectory> prompt_directory,
|
||||||
|
std::unique_ptr<WebClient> web_client)
|
||||||
|
: api_key_(std::move(api_key)),
|
||||||
|
model_(std::move(model)),
|
||||||
|
logger_(std::move(logger)),
|
||||||
|
prompt_directory_(std::move(prompt_directory)),
|
||||||
|
web_client_(std::move(web_client)) {}
|
||||||
@@ -67,7 +67,7 @@ int main(const int argc, char** argv) {
|
|||||||
|
|
||||||
std::unique_ptr<IPromptDirectory> prompt_directory;
|
std::unique_ptr<IPromptDirectory> prompt_directory;
|
||||||
|
|
||||||
if (!options.generator.use_mocked) {
|
if (options.generator.mode != GeneratorMode::kMock) {
|
||||||
try {
|
try {
|
||||||
prompt_directory = std::make_unique<PromptDirectory>(
|
prompt_directory = std::make_unique<PromptDirectory>(
|
||||||
options.pipeline.prompt_dir, log_producer);
|
options.pipeline.prompt_dir, log_producer);
|
||||||
@@ -90,7 +90,7 @@ int main(const int argc, char** argv) {
|
|||||||
di::bind<IPostalCodeService>().to(
|
di::bind<IPostalCodeService>().to(
|
||||||
[options,
|
[options,
|
||||||
&log_producer]() -> std::unique_ptr<IPostalCodeService> {
|
&log_producer]() -> std::unique_ptr<IPostalCodeService> {
|
||||||
if (options.generator.use_mocked) {
|
if (options.generator.mode == GeneratorMode::kMock) {
|
||||||
log_producer->Log(
|
log_producer->Log(
|
||||||
{.level = LogLevel::Info,
|
{.level = LogLevel::Info,
|
||||||
.phase = PipelinePhase::Startup,
|
.phase = PipelinePhase::Startup,
|
||||||
@@ -110,13 +110,13 @@ int main(const int argc, char** argv) {
|
|||||||
di::bind<ICuratedDataService>().to(
|
di::bind<ICuratedDataService>().to(
|
||||||
[options,
|
[options,
|
||||||
&log_producer]() -> std::unique_ptr<ICuratedDataService> {
|
&log_producer]() -> std::unique_ptr<ICuratedDataService> {
|
||||||
if (options.generator.use_mocked) {
|
// if (options.generator.mode == GeneratorMode::kMock) {
|
||||||
log_producer->Log({.level = LogLevel::Info,
|
// log_producer->Log({.level = LogLevel::Info,
|
||||||
.phase = PipelinePhase::Startup,
|
// .phase = PipelinePhase::Startup,
|
||||||
.message = "Curated data: mock"});
|
// .message = "Curated data: mock"});
|
||||||
|
//
|
||||||
return std::make_unique<MockCuratedDataService>();
|
// return std::make_unique<MockCuratedDataService>();
|
||||||
}
|
// }
|
||||||
|
|
||||||
log_producer->Log({.level = LogLevel::Info,
|
log_producer->Log({.level = LogLevel::Info,
|
||||||
.phase = PipelinePhase::Startup,
|
.phase = PipelinePhase::Startup,
|
||||||
@@ -131,13 +131,12 @@ int main(const int argc, char** argv) {
|
|||||||
});
|
});
|
||||||
}),
|
}),
|
||||||
di::bind<IPromptFormatter>().to([options, log_producer] {
|
di::bind<IPromptFormatter>().to([options, log_producer] {
|
||||||
if (options.generator.use_mocked) {
|
if (options.generator.mode != GeneratorMode::kLlama) {
|
||||||
{
|
log_producer->Log(
|
||||||
log_producer->Log(
|
{.level = LogLevel::Info,
|
||||||
{.level = LogLevel::Info,
|
.phase = PipelinePhase::Startup,
|
||||||
.phase = PipelinePhase::Startup,
|
.message = "Prompt formatter: none (not needed outside "
|
||||||
.message = "Prompt formatter: none (mock mode)"});
|
"Llama mode)"});
|
||||||
}
|
|
||||||
return std::unique_ptr<IPromptFormatter>(nullptr);
|
return std::unique_ptr<IPromptFormatter>(nullptr);
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
@@ -151,7 +150,7 @@ int main(const int argc, char** argv) {
|
|||||||
std::make_unique<Gemma4JinjaPromptFormatter>());
|
std::make_unique<Gemma4JinjaPromptFormatter>());
|
||||||
}),
|
}),
|
||||||
di::bind<WebClient>().to([options, log_producer] {
|
di::bind<WebClient>().to([options, log_producer] {
|
||||||
if (options.generator.use_mocked) {
|
if (options.generator.mode == GeneratorMode::kMock) {
|
||||||
{
|
{
|
||||||
log_producer->Log(
|
log_producer->Log(
|
||||||
{.level = LogLevel::Info,
|
{.level = LogLevel::Info,
|
||||||
@@ -171,7 +170,7 @@ int main(const int argc, char** argv) {
|
|||||||
di::bind<IEnrichmentService>().to(
|
di::bind<IEnrichmentService>().to(
|
||||||
[options, &log_producer](
|
[options, &log_producer](
|
||||||
const auto& inj) -> std::unique_ptr<IEnrichmentService> {
|
const auto& inj) -> std::unique_ptr<IEnrichmentService> {
|
||||||
if (options.generator.use_mocked) {
|
if (options.generator.mode == GeneratorMode::kMock) {
|
||||||
log_producer->Log({.level = LogLevel::Info,
|
log_producer->Log({.level = LogLevel::Info,
|
||||||
.phase = PipelinePhase::Startup,
|
.phase = PipelinePhase::Startup,
|
||||||
.message = "Enrichment: mock"});
|
.message = "Enrichment: mock"});
|
||||||
@@ -191,7 +190,7 @@ int main(const int argc, char** argv) {
|
|||||||
[&options, &model_path, &sampling, &prompt_directory,
|
[&options, &model_path, &sampling, &prompt_directory,
|
||||||
&log_producer](
|
&log_producer](
|
||||||
const auto& inj) -> std::unique_ptr<DataGenerator> {
|
const auto& inj) -> std::unique_ptr<DataGenerator> {
|
||||||
if (options.generator.use_mocked) {
|
if (options.generator.mode == GeneratorMode::kMock) {
|
||||||
log_producer->Log({.level = LogLevel::Info,
|
log_producer->Log({.level = LogLevel::Info,
|
||||||
.phase = PipelinePhase::Startup,
|
.phase = PipelinePhase::Startup,
|
||||||
.message = "Generator: mock"});
|
.message = "Generator: mock"});
|
||||||
@@ -199,10 +198,25 @@ int main(const int argc, char** argv) {
|
|||||||
return std::make_unique<MockGenerator>();
|
return std::make_unique<MockGenerator>();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (options.generator.mode == GeneratorMode::kOpenAI) {
|
||||||
|
log_producer->Log(
|
||||||
|
{.level = LogLevel::Info,
|
||||||
|
.phase = PipelinePhase::Startup,
|
||||||
|
.message = std::format(
|
||||||
|
"Generator: OpenAIGenerator | model={}",
|
||||||
|
options.generator.openai_model)});
|
||||||
|
|
||||||
|
return std::make_unique<OpenAIGenerator>(
|
||||||
|
options.generator.openai_api_key,
|
||||||
|
options.generator.openai_model, log_producer,
|
||||||
|
std::move(prompt_directory),
|
||||||
|
inj.template create<std::unique_ptr<WebClient>>());
|
||||||
|
}
|
||||||
|
|
||||||
#ifdef BIERGARTEN_MOCK_ONLY
|
#ifdef BIERGARTEN_MOCK_ONLY
|
||||||
// Unreachable: ParseArguments requires --mocked when
|
// Only reachable when mode == Llama: Mock and OpenAI are
|
||||||
// BIERGARTEN_MOCK_ONLY is compiled in, so LlamaGenerator is
|
// handled above and don't need llama.cpp, but LlamaGenerator
|
||||||
// never constructed and is not linked into this binary.
|
// itself is not linked into a BIERGARTEN_MOCK_ONLY build.
|
||||||
throw std::runtime_error(
|
throw std::runtime_error(
|
||||||
"LlamaGenerator is unavailable in a BIERGARTEN_MOCK_ONLY "
|
"LlamaGenerator is unavailable in a BIERGARTEN_MOCK_ONLY "
|
||||||
"build");
|
"build");
|
||||||
|
|||||||
@@ -30,6 +30,16 @@ static std::string ReadRequiredString(const boost::json::object& object,
|
|||||||
return std::string(text);
|
return std::string(text);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static double ReadRequiredDouble(const boost::json::object& object,
|
||||||
|
const char* key) {
|
||||||
|
const boost::json::value* value = object.if_contains(key);
|
||||||
|
if (value == nullptr || !value->is_double()) {
|
||||||
|
throw std::runtime_error(
|
||||||
|
std::format("Missing or invalid double field: {}", key));
|
||||||
|
}
|
||||||
|
return value->as_double();
|
||||||
|
}
|
||||||
|
|
||||||
static std::vector<std::string> ReadRequiredStringArray(
|
static std::vector<std::string> ReadRequiredStringArray(
|
||||||
const boost::json::object& object, const char* key) {
|
const boost::json::object& object, const char* key) {
|
||||||
const boost::json::value* value = object.if_contains(key);
|
const boost::json::value* value = object.if_contains(key);
|
||||||
@@ -67,7 +77,6 @@ static PostalCodeSpec ReadPostalCodeSpec(const boost::json::object& object) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
boost::json::value ParseJsonFile(const std::filesystem::path& filepath,
|
boost::json::value ParseJsonFile(const std::filesystem::path& filepath,
|
||||||
const char* what) {
|
const char* what) {
|
||||||
std::ifstream input(filepath);
|
std::ifstream input(filepath);
|
||||||
@@ -108,7 +117,6 @@ std::string ReadFirstOfStringArray(const boost::json::object& object,
|
|||||||
throw std::runtime_error(
|
throw std::runtime_error(
|
||||||
std::format("Missing or invalid string array field: {}", key));
|
std::format("Missing or invalid string array field: {}", key));
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
CuratedJsonDataService::CuratedJsonDataService(CuratedDataFilePaths filepaths)
|
CuratedJsonDataService::CuratedJsonDataService(CuratedDataFilePaths filepaths)
|
||||||
@@ -144,6 +152,8 @@ const LocationsList& CuratedJsonDataService::LoadLocations() {
|
|||||||
.iso3166_2 = ReadRequiredString(object, "iso3166_2"),
|
.iso3166_2 = ReadRequiredString(object, "iso3166_2"),
|
||||||
.country = ReadRequiredString(object, "country"),
|
.country = ReadRequiredString(object, "country"),
|
||||||
.iso3166_1 = ReadRequiredString(object, "iso3166_1"),
|
.iso3166_1 = ReadRequiredString(object, "iso3166_1"),
|
||||||
|
.latitude = ReadRequiredDouble(object, "latitude"),
|
||||||
|
.longitude = ReadRequiredDouble(object, "longitude"),
|
||||||
.local_languages = ReadRequiredStringArray(object, "local_languages"),
|
.local_languages = ReadRequiredStringArray(object, "local_languages"),
|
||||||
.postal_code = ReadPostalCodeSpec(object),
|
.postal_code = ReadPostalCodeSpec(object),
|
||||||
});
|
});
|
||||||
@@ -267,4 +277,4 @@ const SurnamesByCountryMap& CuratedJsonDataService::LoadSurnamesByCountry() {
|
|||||||
|
|
||||||
cache_.surnames_by_country = std::move(surnames_by_country);
|
cache_.surnames_by_country = std::move(surnames_by_country);
|
||||||
return cache_.surnames_by_country;
|
return cache_.surnames_by_country;
|
||||||
}
|
}
|
||||||
@@ -66,6 +66,18 @@ sqlite3_int64 SqliteExportService::ResolveCityId(const City& city) {
|
|||||||
.index = sqlite_export_service_internal::kCityIso31661BindIndex,
|
.index = sqlite_export_service_internal::kCityIso31661BindIndex,
|
||||||
.value = city.iso3166_1,
|
.value = city.iso3166_1,
|
||||||
.action = "Failed to bind SQLite city ISO 3166-1 code"});
|
.action = "Failed to bind SQLite city ISO 3166-1 code"});
|
||||||
|
sqlite_export_service_internal::Bind(
|
||||||
|
insert_city_stmt_,
|
||||||
|
sqlite_export_service_internal::BoundParam<double>{
|
||||||
|
.index = sqlite_export_service_internal::kCityLongitudeBindIndex,
|
||||||
|
.value = city.longitude,
|
||||||
|
.action = "Failed to bind SQLite city longitude"});
|
||||||
|
sqlite_export_service_internal::Bind(
|
||||||
|
insert_city_stmt_,
|
||||||
|
sqlite_export_service_internal::BoundParam<double>{
|
||||||
|
.index = sqlite_export_service_internal::kCityLatitudeBindIndex,
|
||||||
|
.value = city.latitude,
|
||||||
|
.action = "Failed to bind SQLite city latitude"});
|
||||||
sqlite_export_service_internal::Bind(
|
sqlite_export_service_internal::Bind(
|
||||||
insert_city_stmt_,
|
insert_city_stmt_,
|
||||||
sqlite_export_service_internal::BoundParam<std::string_view>{
|
sqlite_export_service_internal::BoundParam<std::string_view>{
|
||||||
@@ -165,6 +177,20 @@ uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
|
|||||||
kBreweryAddressPostalCodeBindIndex,
|
kBreweryAddressPostalCodeBindIndex,
|
||||||
.value = brewery.address.postal_code,
|
.value = brewery.address.postal_code,
|
||||||
.action = "Failed to bind SQLite brewery address postal code"});
|
.action = "Failed to bind SQLite brewery address postal code"});
|
||||||
|
sqlite_export_service_internal::Bind(
|
||||||
|
insert_brewery_address_stmt_,
|
||||||
|
sqlite_export_service_internal::BoundParam<double>{
|
||||||
|
.index = sqlite_export_service_internal::
|
||||||
|
kBreweryAddressLongitudeBindIndex,
|
||||||
|
.value = brewery.address.longitude,
|
||||||
|
.action = "Failed to bind SQLite brewery address longitude"});
|
||||||
|
sqlite_export_service_internal::Bind(
|
||||||
|
insert_brewery_address_stmt_,
|
||||||
|
sqlite_export_service_internal::BoundParam<double>{
|
||||||
|
.index = sqlite_export_service_internal::
|
||||||
|
kBreweryAddressLatitudeBindIndex,
|
||||||
|
.value = brewery.address.latitude,
|
||||||
|
.action = "Failed to bind SQLite brewery address latitude"});
|
||||||
|
|
||||||
sqlite_export_service_internal::StepStatement(
|
sqlite_export_service_internal::StepStatement(
|
||||||
db_handle_, insert_brewery_address_stmt_,
|
db_handle_, insert_brewery_address_stmt_,
|
||||||
|
|||||||
@@ -13,6 +13,7 @@
|
|||||||
#include <stdexcept>
|
#include <stdexcept>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <utility>
|
#include <utility>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
#include "services/logging/logger.h"
|
#include "services/logging/logger.h"
|
||||||
|
|
||||||
@@ -67,6 +68,44 @@ std::string HttpWebClient::Get(const std::string& url) {
|
|||||||
return result->body;
|
return result->body;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::string HttpWebClient::Post(
|
||||||
|
const std::string& url, const std::string& body,
|
||||||
|
const std::vector<std::pair<std::string, std::string>>& headers) {
|
||||||
|
const auto [origin, path] = SplitUrl(url);
|
||||||
|
|
||||||
|
httplib::Client client(origin);
|
||||||
|
client.set_follow_location(true);
|
||||||
|
client.set_connection_timeout(kConnectionTimeoutSeconds);
|
||||||
|
client.set_read_timeout(kReadTimeoutSeconds);
|
||||||
|
|
||||||
|
httplib::Headers httplib_headers;
|
||||||
|
for (const auto& [name, value] : headers) {
|
||||||
|
httplib_headers.emplace(name, value);
|
||||||
|
}
|
||||||
|
|
||||||
|
const httplib::Result result =
|
||||||
|
client.Post(path, httplib_headers, body, "application/json");
|
||||||
|
|
||||||
|
if (!result) {
|
||||||
|
throw std::runtime_error(
|
||||||
|
std::format("[HttpWebClient] Request failed for URL: {} — {}", url,
|
||||||
|
httplib::to_string(result.error())));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (result->status < kSuccessMin || result->status >= kSuccessMax) {
|
||||||
|
if (logger_) {
|
||||||
|
logger_->Log({.level = LogLevel::Error,
|
||||||
|
.phase = PipelinePhase::Enrichment,
|
||||||
|
.message = std::format(
|
||||||
|
"[HttpWebClient] Request failed for URL: {}", url)});
|
||||||
|
}
|
||||||
|
throw std::runtime_error(std::format(
|
||||||
|
"[HttpWebClient] HTTP {} for URL: {}", result->status, url));
|
||||||
|
}
|
||||||
|
|
||||||
|
return result->body;
|
||||||
|
}
|
||||||
|
|
||||||
std::string HttpWebClient::EncodeURL(const std::string& value) {
|
std::string HttpWebClient::EncodeURL(const std::string& value) {
|
||||||
return httplib::encode_uri_component(value);
|
return httplib::encode_uri_component(value);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,6 +22,6 @@
|
|||||||
<ProjectReference Include="..\..\Features\Features.Locations\Features.Locations.csproj" />
|
<ProjectReference Include="..\..\Features\Features.Locations\Features.Locations.csproj" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<None CopyToOutputDirectory="PreserveNewest" Include="SeedData\biergarten_seed_2026-07-20T05-46-02.993511Z.sqlite" />
|
<None CopyToOutputDirectory="PreserveNewest" Include="SeedData\open-ai-test2.sqlite" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
</Project>
|
</Project>
|
||||||
|
|||||||
@@ -12,6 +12,6 @@ public class ConnectionStrings
|
|||||||
public static string GetSqlServerConnectionString(IConfiguration configuration) =>
|
public static string GetSqlServerConnectionString(IConfiguration configuration) =>
|
||||||
SqlConnectionStringHelper.BuildConnectionString(configuration);
|
SqlConnectionStringHelper.BuildConnectionString(configuration);
|
||||||
|
|
||||||
public static string SqliteConnectionString =>
|
public static string SqliteConnectionString => "Data Source=SeedData/open-ai-test2.sqlite";
|
||||||
"Data Source=SeedData/biergarten_seed_2026-07-20T05-46-02.993511Z.sqlite";
|
// "Data Source=SeedData/biergarten_seed_2026-07-20T05-46-02.993511Z.sqlite";
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,4 +5,8 @@ public sealed record BreweryAddress
|
|||||||
public required City City { get; init; }
|
public required City City { get; init; }
|
||||||
|
|
||||||
public required string PostalCode { get; init; }
|
public required string PostalCode { get; init; }
|
||||||
|
|
||||||
|
public required double Longitude { get; init; }
|
||||||
|
|
||||||
|
public required double Latitude { get; init; }
|
||||||
}
|
}
|
||||||
|
|||||||
BIN
web/backend/Database/Database.Seed/SeedData/open-ai-test.sqlite
Normal file
BIN
web/backend/Database/Database.Seed/SeedData/open-ai-test.sqlite
Normal file
Binary file not shown.
BIN
web/backend/Database/Database.Seed/SeedData/open-ai-test2.sqlite
Normal file
BIN
web/backend/Database/Database.Seed/SeedData/open-ai-test2.sqlite
Normal file
Binary file not shown.
@@ -54,7 +54,7 @@ public sealed class PipelineSeedDataReader
|
|||||||
{
|
{
|
||||||
const string sql = """
|
const string sql = """
|
||||||
SELECT b.name_en, b.description_en, b.name_local, b.description_local,
|
SELECT b.name_en, b.description_en, b.name_local, b.description_local,
|
||||||
ba.city_id, ba.postal_code
|
ba.city_id, ba.postal_code, ba.longitude, ba.latitude
|
||||||
FROM breweries b
|
FROM breweries b
|
||||||
JOIN brewery_addresses ba ON ba.brewery_id = b.id;
|
JOIN brewery_addresses ba ON ba.brewery_id = b.id;
|
||||||
""";
|
""";
|
||||||
@@ -68,6 +68,8 @@ public sealed class PipelineSeedDataReader
|
|||||||
const int descriptionLocalIndex = 3;
|
const int descriptionLocalIndex = 3;
|
||||||
const int cityIdIndex = 4;
|
const int cityIdIndex = 4;
|
||||||
const int postalCodeIndex = 5;
|
const int postalCodeIndex = 5;
|
||||||
|
const int longitudeIndex = 6;
|
||||||
|
const int latitudeIndex = 7;
|
||||||
|
|
||||||
List<BreweryRecord> records = [];
|
List<BreweryRecord> records = [];
|
||||||
while (await reader.ReadAsync(cancellationToken))
|
while (await reader.ReadAsync(cancellationToken))
|
||||||
@@ -85,6 +87,8 @@ public sealed class PipelineSeedDataReader
|
|||||||
{
|
{
|
||||||
City = cities[reader.GetInt64(cityIdIndex)],
|
City = cities[reader.GetInt64(cityIdIndex)],
|
||||||
PostalCode = reader.GetString(postalCodeIndex),
|
PostalCode = reader.GetString(postalCodeIndex),
|
||||||
|
Longitude = reader.GetDouble(longitudeIndex),
|
||||||
|
Latitude = reader.GetDouble(latitudeIndex),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ public sealed class BreweryPost
|
|||||||
|
|
||||||
// FK References
|
// FK References
|
||||||
public Guid PostedById { get; set; }
|
public Guid PostedById { get; set; }
|
||||||
|
public string PostedBy { get; set; } = string.Empty;
|
||||||
|
|
||||||
// Attributes
|
// Attributes
|
||||||
public string BreweryName { get; set; } = string.Empty;
|
public string BreweryName { get; set; } = string.Empty;
|
||||||
|
|||||||
78
web/generate-env.sh
Executable file
78
web/generate-env.sh
Executable file
@@ -0,0 +1,78 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
#
|
||||||
|
# generate_env.sh
|
||||||
|
#
|
||||||
|
# Generates a Biergarten App environment file with freshly randomized
|
||||||
|
# secrets (DB password, JWT signing secrets, SMTP credentials), based on
|
||||||
|
# the .env.example template. Non-secret configuration values are left
|
||||||
|
# untouched so the file still matches the documented template structure.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# ./generate_env.sh [output_file] [template_file]
|
||||||
|
#
|
||||||
|
# output_file Path to write the generated env file to (default: .env)
|
||||||
|
# template_file Path to the source template (default: .env.example)
|
||||||
|
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
OUTPUT_FILE="${1:-.env}"
|
||||||
|
TEMPLATE_FILE="${2:-.env.example}"
|
||||||
|
|
||||||
|
if ! command -v openssl >/dev/null 2>&1; then
|
||||||
|
echo "Error: openssl is required but was not found on PATH." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ ! -f "$TEMPLATE_FILE" ]]; then
|
||||||
|
echo "Error: template file '$TEMPLATE_FILE' not found." >&2
|
||||||
|
echo "Pass the template path explicitly: ./generate_env.sh <output> <template>" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -f "$OUTPUT_FILE" ]]; then
|
||||||
|
read -r -p "Warning: '$OUTPUT_FILE' already exists. Overwrite? [y/N] " confirm
|
||||||
|
if [[ ! "$confirm" =~ ^[Yy]$ ]]; then
|
||||||
|
echo "Aborted. No file was written."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
gen_jwt_secret() {
|
||||||
|
openssl rand -base64 32 | tr -d '\n'
|
||||||
|
}
|
||||||
|
|
||||||
|
gen_password() {
|
||||||
|
printf '%s!' "$(openssl rand -hex 24)"
|
||||||
|
}
|
||||||
|
|
||||||
|
gen_username() {
|
||||||
|
openssl rand -hex 6
|
||||||
|
}
|
||||||
|
|
||||||
|
DB_PASSWORD_VAL=$(gen_password)
|
||||||
|
ACCESS_TOKEN_SECRET_VAL=$(gen_jwt_secret)
|
||||||
|
REFRESH_TOKEN_SECRET_VAL=$(gen_jwt_secret)
|
||||||
|
CONFIRMATION_TOKEN_SECRET_VAL=$(gen_jwt_secret)
|
||||||
|
SMTP_USERNAME_VAL="smtp_$(gen_username)"
|
||||||
|
SMTP_PASSWORD_VAL=$(gen_password)
|
||||||
|
|
||||||
|
cp "$TEMPLATE_FILE" "$OUTPUT_FILE"
|
||||||
|
|
||||||
|
TMP_FILE="$(mktemp)"
|
||||||
|
sed \
|
||||||
|
-e "s|^DB_PASSWORD=.*|DB_PASSWORD=${DB_PASSWORD_VAL}|" \
|
||||||
|
-e "s|^ACCESS_TOKEN_SECRET=.*|ACCESS_TOKEN_SECRET=${ACCESS_TOKEN_SECRET_VAL}|" \
|
||||||
|
-e "s|^REFRESH_TOKEN_SECRET=.*|REFRESH_TOKEN_SECRET=${REFRESH_TOKEN_SECRET_VAL}|" \
|
||||||
|
-e "s|^CONFIRMATION_TOKEN_SECRET=.*|CONFIRMATION_TOKEN_SECRET=${CONFIRMATION_TOKEN_SECRET_VAL}|" \
|
||||||
|
-e "s|^SMTP_USERNAME=.*|SMTP_USERNAME=${SMTP_USERNAME_VAL}|" \
|
||||||
|
-e "s|^SMTP_PASSWORD=.*|SMTP_PASSWORD=${SMTP_PASSWORD_VAL}|" \
|
||||||
|
"$OUTPUT_FILE" > "$TMP_FILE"
|
||||||
|
mv "$TMP_FILE" "$OUTPUT_FILE"
|
||||||
|
|
||||||
|
chmod 600 "$OUTPUT_FILE"
|
||||||
|
|
||||||
|
echo "Generated '$OUTPUT_FILE' with fresh random secrets (file permissions set to 600)."
|
||||||
|
echo "Reminder: SMTP_USERNAME/SMTP_PASSWORD were randomized as dev placeholders."
|
||||||
|
echo "For staging/production, replace them with real credentials from your SMTP provider."
|
||||||
Reference in New Issue
Block a user