AaronPo/the-biergarten-app
1
0
Fork 0
mirror of https://github.com/aaronpo97/the-biergarten-app.git synced 2026-10-15 01:54:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add reverse geocoding for brewery street addresses (#245)

Browse Source 
* Add geocoding to seed application, remove postal codes from generator

Update comments
This commit is contained in:
Aaron Po
2026-08-23 23:53:56 -04:00
committed by Aaron Po
parent e738fe484a
commit cd5fd2d998
41 changed files with 2966 additions and 2297 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
tooling/pipeline/CMakeLists.txt
View File

@@ -32,7 +32,9 @@ else()
if(CUDAToolkit_FOUND)
message(STATUS "[biergarten] NVIDIA GPU detected — enabling CUDA acceleration.")
set(GGML_CUDA ON CACHE BOOL "Enable CUDA for NVIDIA GPUs" FORCE)
set(CMAKE_CUDA_ARCHITECTURES native)
if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
set(CMAKE_CUDA_ARCHITECTURES native)
endif()
elseif(hip_FOUND OR DEFINED ENV{ROCM_PATH} OR EXISTS "/opt/rocm")
message(STATUS "[biergarten] AMD GPU detected — enabling HIP/ROCm acceleration.")
set(GGML_HIPBLAS ON CACHE BOOL "Enable HIP for AMD GPUs" FORCE)
@@ -149,7 +151,6 @@ add_executable(${PROJECT_NAME}
target_sources(${PROJECT_NAME} PRIVATE
src/main.cc
)
# --- json_handling ---
target_sources(${PROJECT_NAME} PRIVATE
src/services/curated_data/curated_json_data_service.cc
@@ -224,11 +225,6 @@ target_sources(${PROJECT_NAME} PRIVATE
src/services/enrichment/wikipedia/get_summary.cc
)
# --- services: postal_code ---
target_sources(${PROJECT_NAME} PRIVATE
src/services/postal_code/xeger_postal_code_service.cc
)
# --- services: xeger ---
target_sources(${PROJECT_NAME} PRIVATE
src/services/xeger/xeger.cc

5
tooling/pipeline/includes/biergarten_pipeline.h
View File

@@ -61,10 +61,7 @@
#include "services/enrichment/mock_enrichment.h"
#include "services/enrichment/wikipedia_service.h"
// --- services: postal_code ---
#include "services/postal_code/mock_postal_code_service.h"
#include "services/postal_code/postal_code_service.h"
#include "services/postal_code/xeger_postal_code_service.h"
// --- services: xeger ---
#include "services/xeger/xeger.h"
// --- services: logging ---

5
tooling/pipeline/includes/biergarten_pipeline_orchestrator.h
View File

@@ -16,7 +16,6 @@
#include "services/database/export_service.h"
#include "services/enrichment/enrichment_service.h"
#include "services/logging/logger.h"
#include "services/postal_code/postal_code_service.h"
/**
* @brief Main orchestrator for the Biergarten data generation pipeline.
@@ -35,8 +34,6 @@ class BiergartenPipelineOrchestrator {
* @param exporter Database backend for persisting generated records.
* @param curated_data_service Loads curated location, persona, and name
* data used to seed generation.
* @param postal_code_service Resolves a postal code for a city from its
* curated postal-code data.
* @param application_options CLI configuration and paths.
*/
BiergartenPipelineOrchestrator(
@@ -45,7 +42,6 @@ class BiergartenPipelineOrchestrator {
std::unique_ptr<DataGenerator> generator,
std::unique_ptr<IExportService> exporter,
std::unique_ptr<ICuratedDataService> curated_data_service,
std::unique_ptr<IPostalCodeService> postal_code_service,
const ApplicationOptions& application_options);
/**
@@ -82,7 +78,6 @@ class BiergartenPipelineOrchestrator {
*/
std::unique_ptr<IExportService> exporter_;
std::unique_ptr<ICuratedDataService> curated_data_service_;
std::unique_ptr<IPostalCodeService> postal_code_service_;
ApplicationOptions application_options_;

58
tooling/pipeline/includes/biergarten_pipeline_orchestrator/coords.h Normal file
View File

@@ -0,0 +1,58 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_COORDS_H_
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_COORDS_H_
/**
* @file biergarten_pipeline_orchestrator/coords.h
* @brief Shared coordinate jitter helper used to place generated
* brewery/user addresses near a city's centre.
*/
#include <cmath>
#include <numbers>
#include <random>
namespace biergarten_pipeline_orchestrator_internal {
struct Coords {
double longitude;
double latitude;
};
inline constexpr double kEarthRadiusKm = 6371.0;
inline double DegreesToRadians(double degrees) {
return degrees * std::numbers::pi / 180.0;
}
inline double RadiansToDegrees(double radians) {
return radians * 180.0 / std::numbers::pi;
}
/**
* @brief Samples a point uniformly at random within @p distance_km of
* @p centre.
*/
inline Coords GetRandomCoordsWithinRange(const Coords& centre,
const int distance_km,
std::mt19937& rng) {
std::uniform_real_distribution<double> angle_dist(0.0,
2.0 * std::numbers::pi);
std::uniform_real_distribution<double> unit_dist(0.0, 1.0);
const double angle_radians = angle_dist(rng);
// sqrt() keeps points uniformly distributed over the disc's area.
const double radius_km = distance_km * std::sqrt(unit_dist(rng));
const double delta_latitude_degrees =
RadiansToDegrees(radius_km * std::cos(angle_radians) / kEarthRadiusKm);
const double delta_longitude_degrees = RadiansToDegrees(
radius_km * std::sin(angle_radians) /
(kEarthRadiusKm * std::cos(DegreesToRadians(centre.latitude))));
return Coords{.longitude = centre.longitude + delta_longitude_degrees,
.latitude = centre.latitude + delta_latitude_degrees};
}
} // namespace biergarten_pipeline_orchestrator_internal
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_COORDS_H_

14
tooling/pipeline/includes/data_generation/generation_json_validation.h
View File

@@ -3,9 +3,9 @@
/**
* @file data_generation/generation_json_validation.h
* @brief Generator-agnostic helpers for preparing prompt context and
* validating structured JSON output. Shared across all DataGenerator
* implementations (Llama, OpenAI, ...).
* @brief Generator-agnostic helpers, shared across all DataGenerator
* implementations (Llama, OpenAI, ...), for preparing prompt context and
* validating structured JSON output.
*/
#include <cstddef>
@@ -36,11 +36,9 @@ std::optional<std::string> ValidateBreweryJson(const std::string& raw,
BreweryResult& brewery_out);
/**
* @brief Validates and parses user JSON output.
*
* Only populates `username`, `bio`, and `activity_weight` -- `first_name`
* and `last_name` are not LLM-authored and are set separately from the
* sampled Name.
* @brief Validates and parses user JSON output, populating only
* `username`, `bio`, and `activity_weight` (`first_name` and `last_name`
* come from the sampled Name).
*
* @param raw Raw model output.
* @param user_out Parsed user payload.

10
tooling/pipeline/includes/data_generation/openai_generator.h
View File

@@ -18,13 +18,9 @@
/**
* @brief Data generator implementation backed by the OpenAI Chat
* Completions API.
*
* Unlike LlamaGenerator, this generator has no local model state and no
* chat-template formatting step -- Chat Completions takes system/user
* messages directly, and Structured Outputs (response_format: json_schema,
* strict: true) guarantee schema-valid JSON, replacing llama.cpp's GBNF
* grammar constraint.
* Completions API, sending system/user messages directly and using
* Structured Outputs (response_format: json_schema, strict: true) to
* guarantee schema-valid JSON.
*/
class OpenAIGenerator final : public DataGenerator {
public:

11
tooling/pipeline/includes/data_generation/openai_json_schemas.h
View File

@@ -4,14 +4,9 @@
/**
* @file data_generation/openai_json_schemas.h
* @brief JSON Schemas constraining structured output from OpenAIGenerator
* Chat Completions requests, via response_format.json_schema (Structured
* Outputs, strict mode). This is OpenAI's equivalent of the GBNF grammars
* in json_grammars.h -- the API guarantees the response message content is
* a single JSON string that validates against the schema, so there's no
* grammar-level "thought-block" prefix to account for.
*
* OpenAI's strict mode requires "additionalProperties": false on every
* object in the schema, which these already have.
* Chat Completions requests via response_format.json_schema (Structured
* Outputs, strict mode), each setting "additionalProperties": false as
* strict mode requires.
*/
#include <string_view>

12
tooling/pipeline/includes/data_model/generated_models.h
View File

@@ -88,23 +88,23 @@ struct EnrichedCity {
};
/**
* @brief A physical address: the city plus a concrete postal code selected
* from that city's curated postal-code examples.
* @brief A physical address: the city plus a coordinate pair sampled near
* the city's centre.
*/
struct BreweryAddress {
City city;
std::string postal_code{};
double longitude{};
double latitude{};
};
/**
* @brief A physical address for a generated user: the city plus a concrete
* postal code selected from that city's curated postal-code examples.
* @brief A physical address for a generated user: the city plus a
* coordinate pair sampled near the city's centre.
*/
struct UserAddress {
City city;
std::string postal_code{};
double longitude{};
double latitude{};
};
/**

16
tooling/pipeline/includes/services/database/sqlite_statement_helpers.h
View File

@@ -54,7 +54,6 @@ CREATE TABLE IF NOT EXISTS brewery_addresses (
id INTEGER PRIMARY KEY AUTOINCREMENT,
brewery_id INTEGER NOT NULL,
city_id INTEGER NOT NULL,
postal_code TEXT NOT NULL,
longitude REAL NOT NULL,
latitude REAL NOT NULL,
FOREIGN KEY(brewery_id) REFERENCES breweries(id) ON DELETE CASCADE,
@@ -90,7 +89,8 @@ CREATE TABLE IF NOT EXISTS user_addresses (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL,
city_id INTEGER NOT NULL,
postal_code TEXT NOT NULL,
longitude REAL NOT NULL,
latitude REAL NOT NULL,
FOREIGN KEY(user_id) REFERENCES users(id) ON DELETE CASCADE,
FOREIGN KEY(city_id) REFERENCES cities(id) ON DELETE CASCADE
);
@@ -130,10 +130,9 @@ inline constexpr std::string_view kInsertBreweryAddressSql = R"sql(
INSERT INTO brewery_addresses (
brewery_id,
city_id,
postal_code,
longitude,
latitude
) VALUES (?, ?, ?, ?, ?);
) VALUES (?, ?, ?, ?);
)sql";
inline constexpr std::string_view kInsertUserSql = R"sql(
@@ -153,8 +152,9 @@ inline constexpr std::string_view kInsertUserAddressSql = R"sql(
INSERT INTO user_addresses (
user_id,
city_id,
postal_code
) VALUES (?, ?, ?);
longitude,
latitude
) VALUES (?, ?, ?, ?);
)sql";
// sqlite3_bind_*() parameter indices are 1-based, matching the "?"
@@ -182,7 +182,6 @@ enum BreweryBindIndex {
enum BreweryAddressBindIndex {
kBreweryAddressBreweryIdBindIndex = 1,
kBreweryAddressCityIdBindIndex,
kBreweryAddressPostalCodeBindIndex,
kBreweryAddressLongitudeBindIndex,
kBreweryAddressLatitudeBindIndex,
};
@@ -201,7 +200,8 @@ enum UserBindIndex {
enum UserAddressBindIndex {
kUserAddressUserIdBindIndex = 1,
kUserAddressCityIdBindIndex,
kUserAddressPostalCodeBindIndex,
kUserAddressLongitudeBindIndex,
kUserAddressLatitudeBindIndex,
};
SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle,

80
tooling/pipeline/includes/services/postal_code/mock_postal_code_service.h
View File

@@ -1,80 +0,0 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_
/**
* @file services/postal_code/mock_postal_code_service.h
* @brief Data-driven postal code service that selects the first curated
* example for a city and validates it against the city's postal-code regex.
*/
#include <format>
#include <regex>
#include <stdexcept>
#include <string>
#include "services/postal_code/postal_code_service.h"
/**
* @brief Postal code service backed entirely by the curated locations
* dataset: it returns the city's first example postal code after checking it
* against the curated postal-code regex.
*
* A precise city regex is preferred when present; otherwise the permissive
* country format regex is used as the fallback. A city with no examples, or an
* example that fails validation, throws so the caller can skip the city.
*/
class MockPostalCodeService final : public IPostalCodeService {
public:
std::string GeneratePostalCode(const City& city) override {
const PostalCodeSpec& spec = city.postal_code;
if (spec.examples.empty()) {
throw std::runtime_error(std::format(
"No postal-code examples available for city '{}'", city.city));
}
const std::string& candidate = spec.examples.front();
if (!MatchesAnyPattern(candidate, spec)) {
throw std::runtime_error(
std::format("Postal code '{}' for city '{}' does not match the "
"curated postal-code regex",
candidate, city.city));
}
return candidate;
}
private:
/**
* @brief Returns true if @p candidate matches any city regex, or the
* country format regex when no city regex matches. When the spec carries no
* usable regex at all, validation is skipped (returns true).
*/
static bool MatchesAnyPattern(const std::string& candidate,
const PostalCodeSpec& spec) {
bool has_pattern = false;
for (const std::string& pattern : spec.city_regexes) {
if (pattern.empty()) {
continue;
}
has_pattern = true;
if (std::regex_match(candidate, std::regex(pattern))) {
return true;
}
}
if (!spec.country_format_regex.empty()) {
has_pattern = true;
if (std::regex_match(candidate,
std::regex(spec.country_format_regex))) {
return true;
}
}
return !has_pattern;
}
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_

35
tooling/pipeline/includes/services/postal_code/postal_code_service.h
View File

@@ -1,35 +0,0 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_
/**
* @file services/postal_code/postal_code_service.h
* @brief Abstraction for resolving a postal code for a city.
*/
#include <string>
#include "data_model/models.h"
/**
* @brief Interface for services that resolve a postal code for a city.
*/
class IPostalCodeService {
public:
IPostalCodeService() = default;
virtual ~IPostalCodeService() = default;
IPostalCodeService(const IPostalCodeService&) = delete;
IPostalCodeService& operator=(const IPostalCodeService&) = delete;
IPostalCodeService(IPostalCodeService&&) = delete;
IPostalCodeService& operator=(IPostalCodeService&&) = delete;
/**
* @brief Resolves a postal code for @p city.
*
* @param city City whose curated postal-code examples supply the result.
* @return A postal code string, or an empty string if none is available.
*/
virtual std::string GeneratePostalCode(const City& city) = 0;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_

32
tooling/pipeline/includes/services/postal_code/xeger_postal_code_service.h
View File

@@ -1,32 +0,0 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_XEGER_POSTAL_CODE_SERVICE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_XEGER_POSTAL_CODE_SERVICE_H_
/**
* @file services/postal_code/xeger_postal_code_service.h
* @brief Postal code service that generates a fresh, format-conformant code
* per call from the curated regex patterns, instead of returning a fixed
* example.
*/
#include <random>
#include "services/postal_code/postal_code_service.h"
/**
* @brief Postal code service using regex-driven generation (xeger).
*
* Picks among the city's curated `city_regexes` (falls back to
* `country_format_regex`), then generates a random string matching that pattern
* via GenerateStringFromRegex().
*/
class XegerPostalCodeService final : public IPostalCodeService {
public:
XegerPostalCodeService();
std::string GeneratePostalCode(const City& city) override;
private:
std::mt19937 rng_;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_XEGER_POSTAL_CODE_SERVICE_H_

4422
tooling/pipeline/locations.json
View File

File diff suppressed because it is too large Load Diff

9
tooling/pipeline/src/application_options/parse_arguments.cc
View File

@@ -38,9 +38,8 @@ std::optional<ApplicationOptions> ParseArguments(
"Number of layers to offload to GPU");
};
// --mocked, --model, and --openai are mutually exclusive; validation is
// enforced below rather than at registration to produce a clear
// diagnostic message.
// --mocked, --model, and --openai are mutually exclusive, validated below
// to produce a clear diagnostic message.
auto add_generator_options = [&]() -> void {
opt("mocked", prog_opts::bool_switch(),
"Use mocked generator for brewery/user data");
@@ -154,9 +153,7 @@ std::optional<ApplicationOptions> ParseArguments(
return std::nullopt;
}
// Prompt directory is only meaningful for live inference (Llama or
// OpenAI) — the mock generator has no use for it and should not
// require it to be present.
// Prompt directory is required only for live inference (Llama or OpenAI).
if (!use_mocked && options.pipeline.prompt_dir.empty()) {
const std::string msg =
"Invalid arguments: --prompt-dir is required when not using "

2
tooling/pipeline/src/biergarten_pipeline_orchestrator/biergarten_pipeline_orchestrator.cc
View File

@@ -13,12 +13,10 @@ BiergartenPipelineOrchestrator::BiergartenPipelineOrchestrator(
std::unique_ptr<DataGenerator> generator,
std::unique_ptr<IExportService> exporter,
std::unique_ptr<ICuratedDataService> curated_data_service,
std::unique_ptr<IPostalCodeService> postal_code_service,
const ApplicationOptions& application_options)
: logger_(std::move(logger)),
context_service_(std::move(context_service)),
generator_(std::move(generator)),
exporter_(std::move(exporter)),
curated_data_service_(std::move(curated_data_service)),
postal_code_service_(std::move(postal_code_service)),
application_options_(application_options) {}

47
tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_breweries.cc
View File

@@ -4,54 +4,16 @@
*/
#include <chrono>
#include <cmath>
#include <format>
#include <numbers>
#include <optional>
#include <random>
#include "biergarten_pipeline_orchestrator.h"
#include "biergarten_pipeline_orchestrator/coords.h"
#include "services/logging/logger.h"
struct Coords {
double longitude;
double latitude;
};
// @todo find a better home for this
namespace {
constexpr double kEarthRadiusKm = 6371.0;
double DegreesToRadians(double degrees) {
return degrees * std::numbers::pi / 180.0;
}
double RadiansToDegrees(double radians) {
return radians * 180.0 / std::numbers::pi;
}
} // namespace
static Coords GetRandomCoordsWithinRange(const Coords& centre,
const int distance_km,
std::mt19937& rng) {
std::uniform_real_distribution<double> angle_dist(0.0,
2.0 * std::numbers::pi);
std::uniform_real_distribution<double> unit_dist(0.0, 1.0);
const double angle_radians = angle_dist(rng);
// sqrt() keeps points uniformly distributed over the disc's area rather
// than clustering near the centre.
const double radius_km = distance_km * std::sqrt(unit_dist(rng));
const double delta_latitude_degrees =
RadiansToDegrees(radius_km * std::cos(angle_radians) / kEarthRadiusKm);
const double delta_longitude_degrees = RadiansToDegrees(
radius_km * std::sin(angle_radians) /
(kEarthRadiusKm * std::cos(DegreesToRadians(centre.latitude))));
return Coords{.longitude = centre.longitude + delta_longitude_degrees,
.latitude = centre.latitude + delta_latitude_degrees};
}
using biergarten_pipeline_orchestrator_internal::Coords;
using biergarten_pipeline_orchestrator_internal::GetRandomCoordsWithinRange;
void BiergartenPipelineOrchestrator::GenerateBreweries(
std::span<const EnrichedCity> cities) {
@@ -70,8 +32,6 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
try {
const BreweryResult brewery =
generator_->GenerateBrewery(enriched_city);
const std::string postal_code =
postal_code_service_->GeneratePostalCode(enriched_city.location);
constexpr int kMaxDistanceFromCentreKm = 5;
const Coords city_centre{.longitude = enriched_city.location.longitude,
@@ -81,7 +41,6 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
return BreweryRecord{
.address = BreweryAddress{.city = enriched_city.location,
.postal_code = postal_code,
.longitude = brewery_coords.longitude,
.latitude = brewery_coords.latitude},
.brewery = brewery};

17
tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc
View File

@@ -15,9 +15,13 @@
#include <unordered_set>
#include "biergarten_pipeline_orchestrator.h"
#include "biergarten_pipeline_orchestrator/coords.h"
#include "services/curated_data/curated_json_data_service.h"
#include "services/logging/logger.h"
using biergarten_pipeline_orchestrator_internal::Coords;
using biergarten_pipeline_orchestrator_internal::GetRandomCoordsWithinRange;
namespace {
std::string Sanitize(std::string_view value) {
std::string out;
@@ -135,12 +139,17 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
try {
const UserResult user =
generator_->GenerateUser(city, persona, sampled_name);
const std::string postal_code =
postal_code_service_->GeneratePostalCode(city.location);
constexpr int kMaxDistanceFromCentreKm = 5;
const Coords city_centre{.longitude = city.location.longitude,
.latitude = city.location.latitude};
const Coords user_coords = GetRandomCoordsWithinRange(
city_centre, kMaxDistanceFromCentreKm, rng);
return UserRecord{
.address =
UserAddress{.city = city.location, .postal_code = postal_code},
.address = UserAddress{.city = city.location,
.longitude = user_coords.longitude,
.latitude = user_coords.latitude},
.user = user,
.email = BuildEmail(sampled_name, used_email_local_parts),
.date_of_birth = GenerateDateOfBirth(rng),

1
tooling/pipeline/src/biergarten_pipeline_orchestrator/log_results.cc
View File

@@ -26,7 +26,6 @@ void BiergartenPipelineOrchestrator::LogResults() const {
{"country", city.country},
{"state_province", city.state_province},
{"iso3166_2", city.iso3166_2},
{"postal_code", address.postal_code},
{"longitude", address.longitude},
{"latitude", address.latitude},
}}});

4
tooling/pipeline/src/data_generation/generation_json_validation.cc
View File

@@ -58,9 +58,7 @@ std::string CondenseWhitespace(std::string_view text) {
return out;
}
// Guard against truncating in the first half of the string.
// This preserves the critical opening content and avoids cutting critical
// context words early in the region description.
// Keeps truncation past the string's midpoint, preserving the critical opening content.
constexpr size_t kTruncationGuardDivisor = 2;
bool ReadRequiredTrimmedStringField(const boost::json::object& obj,

9
tooling/pipeline/src/data_generation/openai/generate_brewery.cc
View File

@@ -36,10 +36,7 @@ std::string FormatLocalLanguageCodes(
constexpr int kBreweryMaxTokens = 2800;
// Structured outputs already guarantee schema-valid JSON, so retries here
// are only for transient failures or the model returning placeholder text
// -- not for malformed JSON shape, unlike LlamaGenerator's grammar-assisted
// retry loop.
// Structured outputs already guarantee schema-valid JSON, so retries here only cover transient failures or the model returning placeholder text.
constexpr int kMaxAttempts = 2;
} // namespace
@@ -54,9 +51,7 @@ BreweryResult OpenAIGenerator::GenerateBrewery(
FormatLocalLanguageCodes(location.local_languages);
/**
* Load brewery system prompt via the injected prompt directory. The
* same file LlamaGenerator uses -- the prompt content is backend
* agnostic.
* Loads the backend-agnostic brewery system prompt via the injected prompt directory.
*/
const std::string system_prompt =
prompt_directory_->Load("BREWERY_GENERATION");

3
tooling/pipeline/src/data_generation/openai/generate_user.cc
View File

@@ -16,8 +16,7 @@
namespace {
constexpr int kUserMaxTokens = 1200;
// Structured outputs already guarantee schema-valid JSON, so retries here
// are only for transient failures or placeholder text -- not JSON shape.
// Structured outputs already guarantee schema-valid JSON, so retries here only cover transient failures or placeholder text.
constexpr int kMaxAttempts = 2;
} // namespace

38
tooling/pipeline/src/main.cc
View File

@@ -87,36 +87,16 @@ int main(const int argc, char** argv) {
di::bind<ApplicationOptions>().to(options),
di::bind<std::string>().to(model_path),
di::bind<IExportService>().to<SqliteExportService>(),
di::bind<IPostalCodeService>().to(
[options,
&log_producer]() -> std::unique_ptr<IPostalCodeService> {
if (options.generator.mode == GeneratorMode::kMock) {
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Postal code: mock (curated examples)"});
return std::make_unique<MockPostalCodeService>();
}
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message =
"Postal code: Xeger (regex-based generation)"});
return std::make_unique<XegerPostalCodeService>();
}),
di::bind<ICuratedDataService>().to(
[options,
&log_producer]() -> std::unique_ptr<ICuratedDataService> {
// if (options.generator.mode == GeneratorMode::kMock) {
// log_producer->Log({.level = LogLevel::Info,
// .phase = PipelinePhase::Startup,
// .message = "Curated data: mock"});
//
// return std::make_unique<MockCuratedDataService>();
// }
if (options.generator.mode == GeneratorMode::kMock) {
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Curated data: mock"});
return std::make_unique<MockCuratedDataService>();
}
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
@@ -214,9 +194,7 @@ int main(const int argc, char** argv) {
}
#ifdef BIERGARTEN_MOCK_ONLY
// Only reachable when mode == Llama: Mock and OpenAI are
// handled above and don't need llama.cpp, but LlamaGenerator
// itself is not linked into a BIERGARTEN_MOCK_ONLY build.
// Only reachable when mode == Llama, which requires llama.cpp and is excluded from a BIERGARTEN_MOCK_ONLY build.
throw std::runtime_error(
"LlamaGenerator is unavailable in a BIERGARTEN_MOCK_ONLY "
"build");

52
tooling/pipeline/src/services/curated_data/curated_json_data_service.cc
View File

@@ -77,12 +77,11 @@ static PostalCodeSpec ReadPostalCodeSpec(const boost::json::object& object) {
}
namespace {
boost::json::value ParseJsonFile(const std::filesystem::path& filepath,
const char* what) {
boost::json::value ParseJsonFile(const std::filesystem::path& filepath) {
std::ifstream input(filepath);
if (!input.is_open()) {
throw std::runtime_error(
std::format("Failed to open {} file: {}", what, filepath.string()));
std::format("Failed to open file: {}", filepath.string()));
}
std::stringstream buffer;
@@ -92,7 +91,7 @@ boost::json::value ParseJsonFile(const std::filesystem::path& filepath,
boost::json::value root = boost::json::parse(buffer.str(), error);
if (error) {
throw std::runtime_error(
std::format("Failed to parse {} JSON: {}", what, error.message()));
std::format("Failed to parse JSON: {}", error.message()));
}
return root;
}
@@ -126,34 +125,47 @@ const LocationsList& CuratedJsonDataService::LoadLocations() {
if (!cache_.locations.empty()) {
return cache_.locations;
}
boost::json::value root_val =
ParseJsonFile(filepaths_.locations_path);
const boost::json::value root =
ParseJsonFile(filepaths_.locations_path, "locations");
if (!root.is_array()) {
if (!root_val.is_object()) {
throw std::runtime_error(
"Invalid locations JSON: root element must be an array");
"Invalid locations JSON: root element must be an object");
}
LocationsList locations;
const auto& items = root.as_array();
locations.reserve(items.size());
boost::json::object root_obj = root_val.as_object();
for (const auto& item : items) {
if (!root_obj.contains("cities"))
{
throw std::runtime_error(
"Invalid locations JSON: root object must contain a 'cities' key");
}
if (!root_obj["cities"].is_array())
{
throw std::runtime_error(
"Invalid locations JSON: 'cities' key must be an array");
}
LocationsList locations;
const boost::json::array& cities = root_obj["cities"].as_array();
locations.reserve(cities.size());
for (const boost::json::value& item : cities) {
if (!item.is_object()) {
throw std::runtime_error(
"Invalid locations JSON: each entry must be an object");
}
const auto& object = item.as_object();
const boost::json::object& object = item.as_object();
locations.push_back(City{
.city = ReadRequiredString(object, "city"),
.state_province = ReadRequiredString(object, "state_province"),
.iso3166_2 = ReadRequiredString(object, "iso3166_2"),
.country = ReadRequiredString(object, "country"),
.iso3166_1 = ReadRequiredString(object, "iso3166_1"),
.latitude = ReadRequiredDouble(object, "latitude"),
.longitude = ReadRequiredDouble(object, "longitude"),
.latitude = ReadRequiredDouble(object, "latitude"),
.local_languages = ReadRequiredStringArray(object, "local_languages"),
.postal_code = ReadPostalCodeSpec(object),
});
@@ -168,7 +180,7 @@ const PersonasList& CuratedJsonDataService::LoadPersonas() {
}
const boost::json::value root =
ParseJsonFile(filepaths_.personas_path, "personas");
ParseJsonFile(filepaths_.personas_path);
if (!root.is_array()) {
throw std::runtime_error(
@@ -176,10 +188,10 @@ const PersonasList& CuratedJsonDataService::LoadPersonas() {
}
PersonasList personas;
const auto& items = root.as_array();
const boost::json::array& items = root.as_array();
personas.reserve(items.size());
for (const auto& item : items) {
for (const boost::json::value& item : items) {
if (!item.is_object()) {
throw std::runtime_error(
"Invalid personas JSON: each entry must be an object");
@@ -204,7 +216,7 @@ const ForenamesByCountryMap& CuratedJsonDataService::LoadForenamesByCountry() {
}
const boost::json::value root =
ParseJsonFile(filepaths_.forenames_path, "forenames-by-country");
ParseJsonFile(filepaths_.forenames_path);
if (!root.is_object()) {
throw std::runtime_error(
@@ -252,7 +264,7 @@ const SurnamesByCountryMap& CuratedJsonDataService::LoadSurnamesByCountry() {
}
const boost::json::value root =
ParseJsonFile(filepaths_.surnames_path, "surnames-by-country");
ParseJsonFile(filepaths_.surnames_path);
if (!root.is_object()) {
throw std::runtime_error(

44
tooling/pipeline/src/services/postal_code/xeger_postal_code_service.cc
View File

@@ -1,44 +0,0 @@
/**
* @file services/postal_code/xeger_postal_code_service.cc
* @brief XegerPostalCodeService::GeneratePostalCode() implementation.
*/
#include "services/postal_code/xeger_postal_code_service.h"
#include <format>
#include <random>
#include <stdexcept>
#include <string>
#include <string_view>
#include <vector>
#include "services/xeger/xeger.h"
XegerPostalCodeService::XegerPostalCodeService()
: rng_(std::random_device{}()) {}
std::string XegerPostalCodeService::GeneratePostalCode(const City& city) {
const PostalCodeSpec& spec = city.postal_code;
std::vector<std::string_view> city_patterns;
for (const std::string& pattern : spec.city_regexes) {
if (!pattern.empty()) {
city_patterns.push_back(pattern);
}
}
std::string_view pattern;
if (!city_patterns.empty()) {
std::uniform_int_distribution<size_t> dist(0, city_patterns.size() - 1);
pattern = city_patterns[dist(rng_)];
} else if (!spec.country_format_regex.empty()) {
pattern = spec.country_format_regex;
} else {
throw std::runtime_error(std::format(
"XegerPostalCodeService: no postal-code pattern available for "
"city '{}'",
city.city));
}
return GenerateStringFromRegex(pattern, rng_);
}

7
tooling/pipeline/src/services/sqlite/process_record.cc
View File

@@ -170,13 +170,6 @@ uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
sqlite_export_service_internal::kBreweryAddressCityIdBindIndex,
.value = city_id,
.action = "Failed to bind SQLite brewery address city id"});
sqlite_export_service_internal::Bind(
insert_brewery_address_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::
kBreweryAddressPostalCodeBindIndex,
.value = brewery.address.postal_code,
.action = "Failed to bind SQLite brewery address postal code"});
sqlite_export_service_internal::Bind(
insert_brewery_address_stmt_,
sqlite_export_service_internal::BoundParam<double>{

15
tooling/pipeline/src/services/sqlite/process_user_record.cc
View File

@@ -87,11 +87,18 @@ uint64_t SqliteExportService::ProcessRecord(const UserRecord& user) {
.action = "Failed to bind SQLite user address city id"});
sqlite_export_service_internal::Bind(
insert_user_address_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
sqlite_export_service_internal::BoundParam<double>{
.index =
sqlite_export_service_internal::kUserAddressPostalCodeBindIndex,
.value = user.address.postal_code,
.action = "Failed to bind SQLite user address postal code"});
sqlite_export_service_internal::kUserAddressLongitudeBindIndex,
.value = user.address.longitude,
.action = "Failed to bind SQLite user address longitude"});
sqlite_export_service_internal::Bind(
insert_user_address_stmt_,
sqlite_export_service_internal::BoundParam<double>{
.index =
sqlite_export_service_internal::kUserAddressLatitudeBindIndex,
.value = user.address.latitude,
.action = "Failed to bind SQLite user address latitude"});
sqlite_export_service_internal::StepStatement(
db_handle_, insert_user_address_stmt_,