AaronPo/the-biergarten-app
1
0
Fork 0
mirror of https://github.com/aaronpo97/the-biergarten-app.git synced 2026-07-16 09:37:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

style: update include guards and incl statements to follow google style guide

Browse Source
This commit is contained in:
Aaron Po
2026-06-28 13:07:14 -04:00
parent 98f97b8e2c
commit c0ffdb0ec7
28 changed files with 199 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
tooling/pipeline/includes/biergarten_pipeline.h Normal file
View File

@@ -0,0 +1,70 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_
/**
* @file biergarten_pipeline.h
* @brief Master umbrella header — includes every public header in the
* Biergarten pipeline.
*/
// --- orchestrator ---
#include "biergarten_pipeline_orchestrator.h"
// --- concurrency ---
#include "concurrency/bounded_channel.h"
// --- data_generation ---
#include "data_generation/data_generator.h"
#include "data_generation/json_grammars.h"
#include "data_generation/llama_generator.h"
#include "data_generation/llama_generator_helpers.h"
#include "data_generation/mock_generator.h"
#include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h"
#include "data_generation/prompt_formatting/prompt_formatter.h"
// --- data_model ---
#include "data_model/generated_models.h"
#include "data_model/models.h"
#include "data_model/names_by_country.h"
// --- json_handling ---
#include "json_handling/json_loader.h"
#include "json_handling/pretty_print.h"
// --- llama backend ---
#include "llama_backend_state.h"
// --- services: curated_data ---
#include "services/curated_data/curated_data_service.h"
// --- services: database ---
#include "services/database/export_service.h"
#include "services/database/sqlite_connection_helpers.h"
#include "services/database/sqlite_export_service.h"
#include "services/database/sqlite_export_service_helpers.h"
#include "services/database/sqlite_handle_types.h"
#include "services/database/sqlite_statement_helpers.h"
// --- services: datetime ---
#include "services/datetime/date_time_provider.h"
#include "services/datetime/timer.h"
// --- services: enrichment ---
#include "services/enrichment/enrichment_service.h"
#include "services/enrichment/mock_enrichment.h"
#include "services/enrichment/wikipedia_service.h"
// --- services: logging ---
#include "services/logging/log_dispatcher.h"
#include "services/logging/log_entry.h"
#include "services/logging/log_producer.h"
#include "services/logging/logger.h"
// --- services: prompting ---
#include "services/prompting/prompt_directory.h"
// --- web_client ---
#include "web_client/http_web_client.h"
#include "web_client/web_client.h"
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_

13
tooling/pipeline/includes/biergarten_pipeline_orchestrator.h
View File

@@ -1,12 +1,9 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_ #ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_ #define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_
/** /**
* @file biergarten_pipeline_orchestrator.h * @file biergarten_pipeline_orchestrator.h
* @brief Orchestration for end-to-end brewery and user data generation. * @brief Orchestration for end-to-end brewery and user data generation.
*
* Coordinates location loading, enrichment, and generation phases to produce
* a complete dataset, wiring dependencies selected at the composition root.
*/ */
#include <memory> #include <memory>
@@ -103,10 +100,6 @@ class BiergartenPipelineOrchestrator {
* @brief Generate users grounded in sampled names and personas for * @brief Generate users grounded in sampled names and personas for
* enriched cities. * enriched cities.
* *
* Loads personas.json / forenames-by-country.json /
* surnames-by-country.json itself, mirroring how QueryCitiesWithCountries()
* owns its own locations.json load.
*
* @param cities Span of enriched city data. * @param cities Span of enriched city data.
*/ */
void GenerateUsers(std::span<const EnrichedCity> cities); void GenerateUsers(std::span<const EnrichedCity> cities);
@@ -120,4 +113,4 @@ class BiergartenPipelineOrchestrator {
std::vector<UserRecord> generated_users_; std::vector<UserRecord> generated_users_;
}; };
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_ #endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_

2
tooling/pipeline/includes/concurrency/bounded_channel.tcc
View File

@@ -1,4 +1,4 @@
#include "bounded_channel.h" #include "concurrency/bounded_channel.h"
template <typename T> template <typename T>
void BoundedChannel<T>::Send(T item) { void BoundedChannel<T>::Send(T item) {

5
tooling/pipeline/includes/data_generation/llama_generator.h
View File

@@ -1,24 +1,23 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_ #ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_
#define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_ #define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_
#include <filesystem>
/** /**
* @file data_generation/llama_generator.h * @file data_generation/llama_generator.h
* @brief llama.cpp-backed implementation of DataGenerator. * @brief llama.cpp-backed implementation of DataGenerator.
*/ */
#include <cstdint> #include <cstdint>
#include <filesystem>
#include <memory> #include <memory>
#include <random> #include <random>
#include <string> #include <string>
#include <string_view> #include <string_view>
#include "../services/prompting/prompt_directory.h"
#include "data_generation/data_generator.h" #include "data_generation/data_generator.h"
#include "data_generation/prompt_formatting/prompt_formatter.h" #include "data_generation/prompt_formatting/prompt_formatter.h"
#include "data_model/models.h" #include "data_model/models.h"
#include "services/logging/logger.h" #include "services/logging/logger.h"
#include "services/prompting/prompt_directory.h"
struct llama_model; struct llama_model;
struct llama_context; struct llama_context;

3
tooling/pipeline/includes/data_model/models.h
View File

@@ -7,7 +7,6 @@
* inputs. * inputs.
*/ */
#include <boost/program_options.hpp>
#include <cstdint> #include <cstdint>
#include <filesystem> #include <filesystem>
#include <memory> #include <memory>
@@ -16,6 +15,8 @@
#include <string_view> #include <string_view>
#include <vector> #include <vector>
#include <boost/program_options.hpp>
class ILogger; class ILogger;
namespace prog_opts = boost::program_options; namespace prog_opts = boost::program_options;

5
tooling/pipeline/includes/json_handling/pretty_print.h
View File

@@ -9,10 +9,11 @@
* readable output. Adapted from Boost JSON library examples. * readable output. Adapted from Boost JSON library examples.
*/ */
#include <boost/json.hpp>
#include <ostream> #include <ostream>
#include <string> #include <string>
#include <boost/json.hpp>
/** /**
* @brief Pretty-prints a JSON value to an output stream with indentation. * @brief Pretty-prints a JSON value to an output stream with indentation.
* *
@@ -106,4 +107,4 @@ inline void PrettyPrint(std::ostream& outstream,
} }
} }
#endif #endif // BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_PRETTY_PRINT_H_

6
tooling/pipeline/includes/services/database/sqlite_connection_helpers.h
View File

@@ -6,13 +6,13 @@
* @brief Declarations for connection-level SQLite helper functions. * @brief Declarations for connection-level SQLite helper functions.
*/ */
#include <sqlite3.h>
#include <filesystem> #include <filesystem>
#include <string> #include <string>
#include <string_view> #include <string_view>
#include "sqlite_handle_types.h" #include <sqlite3.h>
#include "services/database/sqlite_handle_types.h"
namespace sqlite_export_service_internal { namespace sqlite_export_service_internal {

6
tooling/pipeline/includes/services/database/sqlite_export_service.h
View File

@@ -11,10 +11,10 @@
#include <string> #include <string>
#include <unordered_map> #include <unordered_map>
#include "../datetime/date_time_provider.h"
#include "data_model/models.h" #include "data_model/models.h"
#include "export_service.h" #include "services/database/export_service.h"
#include "sqlite_export_service_helpers.h" #include "services/database/sqlite_export_service_helpers.h"
#include "services/datetime/date_time_provider.h"
/** /**
* @brief Persists generated brewery records into a fresh SQLite database. * @brief Persists generated brewery records into a fresh SQLite database.

6
tooling/pipeline/includes/services/database/sqlite_export_service_helpers.h
View File

@@ -3,8 +3,8 @@
/* Umbrella header for backward compatibility. */ /* Umbrella header for backward compatibility. */
#include "sqlite_connection_helpers.h" #include "services/database/sqlite_connection_helpers.h"
#include "sqlite_handle_types.h" #include "services/database/sqlite_handle_types.h"
#include "sqlite_statement_helpers.h" #include "services/database/sqlite_statement_helpers.h"
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATABASE_SQLITE_EXPORT_SERVICE_HELPERS_H_ #endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATABASE_SQLITE_EXPORT_SERVICE_HELPERS_H_

4
tooling/pipeline/includes/services/database/sqlite_handle_types.h
View File

@@ -5,11 +5,11 @@
* Shared handle and parameter type declarations used by SQLite helper units. * Shared handle and parameter type declarations used by SQLite helper units.
*/ */
#include <sqlite3.h>
#include <memory> #include <memory>
#include <string_view> #include <string_view>
#include <sqlite3.h>
namespace sqlite_export_service_internal { namespace sqlite_export_service_internal {
struct SqliteDatabaseDeleter { struct SqliteDatabaseDeleter {

56
tooling/pipeline/includes/services/database/sqlite_statement_helpers.h
View File

@@ -7,13 +7,13 @@
* constants. * constants.
*/ */
#include <sqlite3.h>
#include <string> #include <string>
#include <string_view> #include <string_view>
#include <vector> #include <vector>
#include "sqlite_handle_types.h" #include <sqlite3.h>
#include "services/database/sqlite_handle_types.h"
namespace sqlite_export_service_internal { namespace sqlite_export_service_internal {
@@ -109,30 +109,36 @@ INSERT INTO users (
// sqlite3_bind_*() parameter indices are 1-based, matching the "?" // sqlite3_bind_*() parameter indices are 1-based, matching the "?"
// placeholder order in the SQL above. // placeholder order in the SQL above.
inline constexpr int kLocationCityBindIndex = 1; enum eLocationBindIndex {
inline constexpr int kLocationStateProvinceBindIndex = 2; kLocationCityBindIndex = 1,
inline constexpr int kLocationIso31662BindIndex = 3; kLocationStateProvinceBindIndex,
inline constexpr int kLocationCountryBindIndex = 4; kLocationIso31662BindIndex,
inline constexpr int kLocationIso31661BindIndex = 5; kLocationCountryBindIndex,
inline constexpr int kLocationLanguagesBindIndex = 6; kLocationIso31661BindIndex,
inline constexpr int kLocationLatitudeBindIndex = 7; kLocationLanguagesBindIndex,
inline constexpr int kLocationLongitudeBindIndex = 8; kLocationLatitudeBindIndex,
kLocationLongitudeBindIndex,
};
inline constexpr int kBreweryLocationIdBindIndex = 1; enum BreweryBindIndex {
inline constexpr int kBreweryEnglishNameBindIndex = 2; kBreweryLocationIdBindIndex = 1,
inline constexpr int kBreweryEnglishDescriptionBindIndex = 3; kBreweryEnglishNameBindIndex,
inline constexpr int kBreweryLocalNameBindIndex = 4; kBreweryEnglishDescriptionBindIndex,
inline constexpr int kBreweryLocalDescriptionBindIndex = 5; kBreweryLocalNameBindIndex,
kBreweryLocalDescriptionBindIndex,
};
inline constexpr int kUserLocationIdBindIndex = 1; enum UserBindIndex {
inline constexpr int kUserFirstNameBindIndex = 2; kUserLocationIdBindIndex = 1,
inline constexpr int kUserLastNameBindIndex = 3; kUserFirstNameBindIndex,
inline constexpr int kUserGenderBindIndex = 4; kUserLastNameBindIndex,
inline constexpr int kUserUsernameBindIndex = 5; kUserGenderBindIndex,
inline constexpr int kUserBioBindIndex = 6; kUserUsernameBindIndex,
inline constexpr int kUserActivityWeightBindIndex = 7; kUserBioBindIndex,
inline constexpr int kUserEmailBindIndex = 8; kUserActivityWeightBindIndex,
inline constexpr int kUserDateOfBirthBindIndex = 9; kUserEmailBindIndex,
kUserDateOfBirthBindIndex,
};
SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle, SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle,
std::string_view sql, std::string_view sql,

2
tooling/pipeline/includes/services/enrichment/mock_enrichment.h
View File

@@ -8,7 +8,7 @@
#include <string> #include <string>
#include "enrichment_service.h" #include "services/enrichment/enrichment_service.h"
/** /**
* @brief Enrichment service that returns no context for any location. * @brief Enrichment service that returns no context for any location.

2
tooling/pipeline/includes/services/enrichment/wikipedia_service.h
View File

@@ -11,7 +11,7 @@
#include <string_view> #include <string_view>
#include <unordered_map> #include <unordered_map>
#include "enrichment_service.h" #include "services/enrichment/enrichment_service.h"
#include "services/logging/logger.h" #include "services/logging/logger.h"
#include "web_client/web_client.h" #include "web_client/web_client.h"

6
tooling/pipeline/includes/services/logging/log_producer.h
View File

@@ -6,8 +6,8 @@
* a bounded channel for later processing by the dispatcher. * a bounded channel for later processing by the dispatcher.
*/ */
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_ #ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_ #define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_
#include <string_view> #include <string_view>
@@ -50,4 +50,4 @@ class LogProducer final : public ILogger {
BoundedChannel<LogEntry>& channel_; BoundedChannel<LogEntry>& channel_;
}; };
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_ #endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_

2
tooling/pipeline/includes/web_client/http_web_client.h
View File

@@ -52,4 +52,4 @@ class HttpWebClient final : public WebClient {
std::shared_ptr<ILogger> logger_; std::shared_ptr<ILogger> logger_;
}; };
#endif #endif // BIERGARTEN_PIPELINE_INCLUDES_WEB_CLIENT_HTTP_WEB_CLIENT_H_

13
tooling/pipeline/src/biergarten_pipeline_orchestrator/biergarten_pipeline_orchestrator.cc
View File

@@ -14,10 +14,9 @@ BiergartenPipelineOrchestrator::BiergartenPipelineOrchestrator(
std::unique_ptr<IExportService> exporter, std::unique_ptr<IExportService> exporter,
std::unique_ptr<ICuratedDataService> curated_data_service, std::unique_ptr<ICuratedDataService> curated_data_service,
const ApplicationOptions& application_options) const ApplicationOptions& application_options)
: logger_(std::move(logger)), : logger_(std::move(logger)),
context_service_(std::move(context_service)), context_service_(std::move(context_service)),
generator_(std::move(generator)), generator_(std::move(generator)),
exporter_(std::move(exporter)), exporter_(std::move(exporter)),
curated_data_service_(std::move(curated_data_service)), curated_data_service_(std::move(curated_data_service)),
application_options_(application_options) { application_options_(application_options) {}
}

36
tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc
View File

@@ -24,7 +24,7 @@ std::string Sanitize(std::string_view value) {
for (const char character : value) { for (const char character : value) {
if (std::isalnum(static_cast<unsigned char>(character)) != 0) { if (std::isalnum(static_cast<unsigned char>(character)) != 0) {
out.push_back(static_cast<char>( out.push_back(static_cast<char>(
std::tolower(static_cast<unsigned char>(character)))); std::tolower(static_cast<unsigned char>(character))));
} }
} }
return out; return out;
@@ -69,7 +69,7 @@ std::string GenerateDateOfBirth(std::mt19937& rng) {
static_cast<unsigned>(birth_ymd.month()), static_cast<unsigned>(birth_ymd.month()),
static_cast<unsigned>(birth_ymd.day())); static_cast<unsigned>(birth_ymd.day()));
} }
} // namespace } // namespace
void BiergartenPipelineOrchestrator::GenerateUsers( void BiergartenPipelineOrchestrator::GenerateUsers(
std::span<const EnrichedCity> cities) { std::span<const EnrichedCity> cities) {
@@ -86,8 +86,8 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
} }
const NamesByCountry names_by_country = const NamesByCountry names_by_country =
curated_data_service_->LoadNamesByCountry( curated_data_service_->LoadNamesByCountry("forenames-by-country.json",
"forenames-by-country.json", "surnames-by-country.json"); "surnames-by-country.json");
std::mt19937 rng(std::random_device{}()); std::mt19937 rng(std::random_device{}());
std::uniform_int_distribution<size_t> persona_dist(0, personas.size() - 1); std::uniform_int_distribution<size_t> persona_dist(0, personas.size() - 1);
@@ -98,8 +98,8 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
const auto generate_record = const auto generate_record =
[this, &rng, &skipped_count]( [this, &rng, &skipped_count](
const EnrichedCity& city, const UserPersona& persona, const EnrichedCity& city, const UserPersona& persona,
const Name& sampled_name) -> std::optional<UserRecord> { const Name& sampled_name) -> std::optional<UserRecord> {
try { try {
std::unordered_set<std::string> used_email_local_parts; std::unordered_set<std::string> used_email_local_parts;
@@ -125,19 +125,19 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
}; };
const auto export_record = [this, const auto export_record = [this,
&export_failed_count](const UserRecord& record) { &export_failed_count](const UserRecord& record) {
try { try {
exporter_->ProcessRecord(record); exporter_->ProcessRecord(record);
} catch (const std::exception& export_exception) { } catch (const std::exception& export_exception) {
++export_failed_count; ++export_failed_count;
logger_->Log( logger_->Log(
{.level = LogLevel::Warn, {.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration, .phase = PipelinePhase::UserGeneration,
.message = std::format("[Pipeline] Generated user for '{}' ({}) but " .message = std::format("[Pipeline] Generated user for '{}' ({}) but "
"SQLite export failed: {}", "SQLite export failed: {}",
record.location.city, record.location.country, record.location.city, record.location.country,
export_exception.what())}); export_exception.what())});
} }
}; };
@@ -171,11 +171,11 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
if (skipped_count > 0) { if (skipped_count > 0) {
logger_->Log( logger_->Log(
{.level = LogLevel::Warn, {.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration, .phase = PipelinePhase::UserGeneration,
.message = std::format( .message = std::format(
"[Pipeline] Skipped {} city/cities during user generation", "[Pipeline] Skipped {} city/cities during user generation",
skipped_count)}); skipped_count)});
} }
if (export_failed_count > 0) { if (export_failed_count > 0) {

5
tooling/pipeline/src/biergarten_pipeline_orchestrator/log_results.cc
View File

@@ -3,12 +3,13 @@
* @brief BiergartenPipelineOrchestrator::LogResults() implementation. * @brief BiergartenPipelineOrchestrator::LogResults() implementation.
*/ */
#include <boost/json/array.hpp>
#include <chrono> #include <chrono>
#include <format> #include <format>
#include "../../includes/json_handling/pretty_print.h" #include <boost/json/array.hpp>
#include "biergarten_pipeline_orchestrator.h" #include "biergarten_pipeline_orchestrator.h"
#include "json_handling/pretty_print.h"
#include "services/logging/logger.h" #include "services/logging/logger.h"
void BiergartenPipelineOrchestrator::LogResults() const { void BiergartenPipelineOrchestrator::LogResults() const {

6
tooling/pipeline/src/data_generation/llama/helpers.cc
View File

@@ -6,7 +6,6 @@
#include <algorithm> #include <algorithm>
#include <array> #include <array>
#include <boost/json.hpp>
#include <cctype> #include <cctype>
#include <optional> #include <optional>
#include <span> #include <span>
@@ -15,8 +14,11 @@
#include <string_view> #include <string_view>
#include <vector> #include <vector>
#include <boost/json.hpp>
#include <llama.h>
#include "data_generation/llama_generator_helpers.h" #include "data_generation/llama_generator_helpers.h"
#include "llama.h"
namespace { namespace {
/** /**
* String trimming: removes leading and trailing whitespace * String trimming: removes leading and trailing whitespace

3
tooling/pipeline/src/data_generation/llama/infer.cc
View File

@@ -14,9 +14,10 @@
#include <string_view> #include <string_view>
#include <vector> #include <vector>
#include <llama.h>
#include "data_generation/llama_generator.h" #include "data_generation/llama_generator.h"
#include "data_generation/llama_generator_helpers.h" #include "data_generation/llama_generator_helpers.h"
#include "llama.h"
static constexpr size_t kPromptTokenSlack = 8; static constexpr size_t kPromptTokenSlack = 8;
// Minimum tokens to keep when using top-p sampling. Ensures at least one // Minimum tokens to keep when using top-p sampling. Ensures at least one

3
tooling/pipeline/src/data_generation/llama/llama_generator.cc
View File

@@ -11,8 +11,9 @@
#include <stdexcept> #include <stdexcept>
#include <string> #include <string>
#include <llama.h>
#include "data_model/models.h" #include "data_model/models.h"
#include "llama.h"
static constexpr uint32_t kMaxContextSize = 32768U; static constexpr uint32_t kMaxContextSize = 32768U;

5
tooling/pipeline/src/data_generation/llama/load.cc
View File

@@ -10,9 +10,10 @@
#include <string> #include <string>
#include <utility> #include <utility>
#include <ggml-backend.h>
#include <llama.h>
#include "data_generation/llama_generator.h" #include "data_generation/llama_generator.h"
#include "ggml-backend.h"
#include "llama.h"
// Maximum batch size for decode operations. Capping the batch prevents // Maximum batch size for decode operations. Capping the batch prevents
// excessive memory allocation while maintaining inference performance. // excessive memory allocation while maintaining inference performance.

4
tooling/pipeline/src/json_handling/json_loader.cc
View File

@@ -6,7 +6,6 @@
#include "json_handling/json_loader.h" #include "json_handling/json_loader.h"
#include <boost/json.hpp>
#include <format> #include <format>
#include <fstream> #include <fstream>
#include <iostream> #include <iostream>
@@ -17,6 +16,8 @@
#include <unordered_map> #include <unordered_map>
#include <utility> #include <utility>
#include <boost/json.hpp>
#include "services/logging/logger.h" #include "services/logging/logger.h"
static std::string ReadRequiredString(const boost::json::object& object, static std::string ReadRequiredString(const boost::json::object& object,
@@ -137,7 +138,6 @@ std::vector<Location> JsonLoader::LoadLocations(
.longitude = ReadRequiredNumber(object, "longitude"), .longitude = ReadRequiredNumber(object, "longitude"),
}); });
} }
return locations; return locations;
} }

65
tooling/pipeline/src/main.cc
View File

@@ -4,11 +4,6 @@
* initializes shared infrastructure, and executes the pipeline entry flow. * initializes shared infrastructure, and executes the pipeline entry flow.
*/ */
#include <spdlog/fmt/fmt.h>
#include <spdlog/spdlog.h>
#include <boost/di.hpp>
#include <boost/program_options.hpp>
#include <chrono> #include <chrono>
#include <exception> #include <exception>
#include <format> #include <format>
@@ -18,27 +13,12 @@
#include <string> #include <string>
#include <thread> #include <thread>
#include "biergarten_pipeline_orchestrator.h" #include <boost/di.hpp>
#include "concurrency/bounded_channel.h" #include <boost/program_options.hpp>
#include "data_generation/llama_generator.h" #include <spdlog/fmt/fmt.h>
#include "data_generation/mock_generator.h" #include <spdlog/spdlog.h>
#include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h"
#include "data_model/models.h" #include "biergarten_pipeline.h"
#include "json_handling/json_loader.h"
#include "llama_backend_state.h"
#include "services/curated_data/curated_data_service.h"
#include "services/database/export_service.h"
#include "services/database/sqlite_export_service.h"
#include "services/datetime/timer.h"
#include "services/enrichment/enrichment_service.h"
#include "services/enrichment/mock_enrichment.h"
#include "services/enrichment/wikipedia_service.h"
#include "services/logging/log_dispatcher.h"
#include "services/logging/log_entry.h"
#include "services/logging/log_producer.h"
#include "services/logging/logger.h"
#include "services/prompting/prompt_directory.h"
#include "web_client/http_web_client.h"
namespace di = boost::di; namespace di = boost::di;
@@ -49,7 +29,7 @@ int main(const int argc, char** argv) {
spdlog::set_pattern("│ %Y-%m-%d %H:%M:%S.%e │ %^%-7l%$ │ %v"); spdlog::set_pattern("│ %Y-%m-%d %H:%M:%S.%e │ %^%-7l%$ │ %v");
BoundedChannel<LogEntry> log_channel(kLogMaxCount); BoundedChannel<LogEntry> log_channel(kLogMaxCount);
auto log_dispatcher = // auto log_dispatcher = //
std::make_unique<LogDispatcher>(log_channel); std::make_unique<LogDispatcher>(log_channel);
std::shared_ptr<ILogger> log_producer = std::shared_ptr<ILogger> log_producer =
std::make_shared<LogProducer>(log_channel); std::make_shared<LogProducer>(log_channel);
@@ -111,17 +91,17 @@ int main(const int argc, char** argv) {
if (options.generator.use_mocked) { if (options.generator.use_mocked) {
{ {
log_producer->Log( log_producer->Log(
{.level = LogLevel::Info, {.level = LogLevel::Info,
.phase = PipelinePhase::Startup, .phase = PipelinePhase::Startup,
.message = "Prompt formatter: none (mock mode)"}); .message = "Prompt formatter: none (mock mode)"});
} }
return std::unique_ptr<IPromptFormatter>(nullptr); return std::unique_ptr<IPromptFormatter>(nullptr);
} }
{ {
log_producer->Log( log_producer->Log(
{.level = LogLevel::Info, {.level = LogLevel::Info,
.phase = PipelinePhase::Startup, .phase = PipelinePhase::Startup,
.message = "Prompt formatter: Gemma4JinjaPromptFormatter"}); .message = "Prompt formatter: Gemma4JinjaPromptFormatter"});
} }
return std::unique_ptr<IPromptFormatter>( return std::unique_ptr<IPromptFormatter>(
std::make_unique<Gemma4JinjaPromptFormatter>()); std::make_unique<Gemma4JinjaPromptFormatter>());
@@ -145,7 +125,7 @@ int main(const int argc, char** argv) {
}), }),
di::bind<IEnrichmentService>().to( di::bind<IEnrichmentService>().to(
[options, &log_producer]( [options, &log_producer](
const auto& inj) -> std::unique_ptr<IEnrichmentService> { const auto& inj) -> std::unique_ptr<IEnrichmentService> {
if (options.generator.use_mocked) { if (options.generator.use_mocked) {
log_producer->Log({.level = LogLevel::Info, log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup, .phase = PipelinePhase::Startup,
@@ -164,8 +144,7 @@ int main(const int argc, char** argv) {
}), }),
di::bind<DataGenerator>().to( di::bind<DataGenerator>().to(
[&options, &model_path, &sampling, &prompt_directory, [&options, &model_path, &sampling, &prompt_directory,
&log_producer &log_producer](const auto& inj) -> std::unique_ptr<DataGenerator> {
](const auto& inj) -> std::unique_ptr<DataGenerator> {
if (options.generator.use_mocked) { if (options.generator.use_mocked) {
log_producer->Log({.level = LogLevel::Info, log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup, .phase = PipelinePhase::Startup,
@@ -175,13 +154,13 @@ int main(const int argc, char** argv) {
} }
log_producer->Log( log_producer->Log(
{.level = LogLevel::Info, {.level = LogLevel::Info,
.phase = PipelinePhase::Startup, .phase = PipelinePhase::Startup,
.message = std::format( .message = std::format(
"Generator: LlamaGenerator | model={} | " "Generator: LlamaGenerator | model={} | "
"temp={:.2f} top_p={:.2f} top_k={} n_ctx={} seed={}", "temp={:.2f} top_p={:.2f} top_k={} n_ctx={} seed={}",
model_path, sampling.temperature, sampling.top_p, model_path, sampling.temperature, sampling.top_p,
sampling.top_k, sampling.n_ctx, sampling.seed)}); sampling.top_k, sampling.n_ctx, sampling.seed)});
return std::make_unique<LlamaGenerator>( return std::make_unique<LlamaGenerator>(
options, model_path, log_producer, options, model_path, log_producer,

3
tooling/pipeline/src/services/enrichment/wikipedia/fetch_extract.cc
View File

@@ -3,13 +3,14 @@
* @brief WikipediaEnrichmentService::FetchExtract() implementation. * @brief WikipediaEnrichmentService::FetchExtract() implementation.
*/ */
#include <boost/json.hpp>
#include <chrono> #include <chrono>
#include <format> #include <format>
#include <string> #include <string>
#include <string_view> #include <string_view>
#include <thread> #include <thread>
#include <boost/json.hpp>
#include "services/enrichment/wikipedia_service.h" #include "services/enrichment/wikipedia_service.h"
using namespace boost; using namespace boost;

4
tooling/pipeline/src/services/logging/log_dispatcher.cc
View File

@@ -6,10 +6,10 @@
*/ */
#include "services/logging/log_dispatcher.h" #include "services/logging/log_dispatcher.h"
#include <spdlog/spdlog.h>
#include <string> #include <string>
#include <spdlog/spdlog.h>
#include "concurrency/bounded_channel.h" #include "concurrency/bounded_channel.h"
#include "services/logging/log_entry.h" #include "services/logging/log_entry.h"

3
tooling/pipeline/src/services/sqlite/helpers/sqlite_statement_helpers.cc
View File

@@ -1,11 +1,12 @@
#include "services/database/sqlite_statement_helpers.h" #include "services/database/sqlite_statement_helpers.h"
#include <boost/json.hpp>
#include <cstring> #include <cstring>
#include <limits> #include <limits>
#include <memory> #include <memory>
#include <stdexcept> #include <stdexcept>
#include <boost/json.hpp>
#include "services/database/sqlite_connection_helpers.h" #include "services/database/sqlite_connection_helpers.h"
namespace sqlite_export_service_internal { namespace sqlite_export_service_internal {

4
tooling/pipeline/src/web_client/http_web_client.cc
View File

@@ -5,8 +5,6 @@
#include "web_client/http_web_client.h" #include "web_client/http_web_client.h"
#include <httplib.h>
#include <chrono> #include <chrono>
#include <format> #include <format>
#include <regex> #include <regex>
@@ -14,6 +12,8 @@
#include <string> #include <string>
#include <utility> #include <utility>
#include <httplib.h>
#include "services/logging/logger.h" #include "services/logging/logger.h"
namespace { namespace {