mirror of
https://github.com/aaronpo97/the-biergarten-app.git
synced 2026-07-16 09:37:23 +00:00
style: update include guards and incl statements to follow google style guide
This commit is contained in:
70
tooling/pipeline/includes/biergarten_pipeline.h
Normal file
70
tooling/pipeline/includes/biergarten_pipeline.h
Normal file
@@ -0,0 +1,70 @@
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_
|
||||
|
||||
/**
|
||||
* @file biergarten_pipeline.h
|
||||
* @brief Master umbrella header — includes every public header in the
|
||||
* Biergarten pipeline.
|
||||
*/
|
||||
|
||||
// --- orchestrator ---
|
||||
#include "biergarten_pipeline_orchestrator.h"
|
||||
|
||||
// --- concurrency ---
|
||||
#include "concurrency/bounded_channel.h"
|
||||
|
||||
// --- data_generation ---
|
||||
#include "data_generation/data_generator.h"
|
||||
#include "data_generation/json_grammars.h"
|
||||
#include "data_generation/llama_generator.h"
|
||||
#include "data_generation/llama_generator_helpers.h"
|
||||
#include "data_generation/mock_generator.h"
|
||||
#include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h"
|
||||
#include "data_generation/prompt_formatting/prompt_formatter.h"
|
||||
|
||||
// --- data_model ---
|
||||
#include "data_model/generated_models.h"
|
||||
#include "data_model/models.h"
|
||||
#include "data_model/names_by_country.h"
|
||||
|
||||
// --- json_handling ---
|
||||
#include "json_handling/json_loader.h"
|
||||
#include "json_handling/pretty_print.h"
|
||||
|
||||
// --- llama backend ---
|
||||
#include "llama_backend_state.h"
|
||||
|
||||
// --- services: curated_data ---
|
||||
#include "services/curated_data/curated_data_service.h"
|
||||
|
||||
// --- services: database ---
|
||||
#include "services/database/export_service.h"
|
||||
#include "services/database/sqlite_connection_helpers.h"
|
||||
#include "services/database/sqlite_export_service.h"
|
||||
#include "services/database/sqlite_export_service_helpers.h"
|
||||
#include "services/database/sqlite_handle_types.h"
|
||||
#include "services/database/sqlite_statement_helpers.h"
|
||||
|
||||
// --- services: datetime ---
|
||||
#include "services/datetime/date_time_provider.h"
|
||||
#include "services/datetime/timer.h"
|
||||
|
||||
// --- services: enrichment ---
|
||||
#include "services/enrichment/enrichment_service.h"
|
||||
#include "services/enrichment/mock_enrichment.h"
|
||||
#include "services/enrichment/wikipedia_service.h"
|
||||
|
||||
// --- services: logging ---
|
||||
#include "services/logging/log_dispatcher.h"
|
||||
#include "services/logging/log_entry.h"
|
||||
#include "services/logging/log_producer.h"
|
||||
#include "services/logging/logger.h"
|
||||
|
||||
// --- services: prompting ---
|
||||
#include "services/prompting/prompt_directory.h"
|
||||
|
||||
// --- web_client ---
|
||||
#include "web_client/http_web_client.h"
|
||||
#include "web_client/web_client.h"
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_
|
||||
@@ -1,12 +1,9 @@
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_
|
||||
|
||||
/**
|
||||
* @file biergarten_pipeline_orchestrator.h
|
||||
* @brief Orchestration for end-to-end brewery and user data generation.
|
||||
*
|
||||
* Coordinates location loading, enrichment, and generation phases to produce
|
||||
* a complete dataset, wiring dependencies selected at the composition root.
|
||||
*/
|
||||
|
||||
#include <memory>
|
||||
@@ -103,10 +100,6 @@ class BiergartenPipelineOrchestrator {
|
||||
* @brief Generate users grounded in sampled names and personas for
|
||||
* enriched cities.
|
||||
*
|
||||
* Loads personas.json / forenames-by-country.json /
|
||||
* surnames-by-country.json itself, mirroring how QueryCitiesWithCountries()
|
||||
* owns its own locations.json load.
|
||||
*
|
||||
* @param cities Span of enriched city data.
|
||||
*/
|
||||
void GenerateUsers(std::span<const EnrichedCity> cities);
|
||||
@@ -120,4 +113,4 @@ class BiergartenPipelineOrchestrator {
|
||||
std::vector<UserRecord> generated_users_;
|
||||
};
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#include "bounded_channel.h"
|
||||
#include "concurrency/bounded_channel.h"
|
||||
|
||||
template <typename T>
|
||||
void BoundedChannel<T>::Send(T item) {
|
||||
|
||||
@@ -1,24 +1,23 @@
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_
|
||||
|
||||
#include <filesystem>
|
||||
|
||||
/**
|
||||
* @file data_generation/llama_generator.h
|
||||
* @brief llama.cpp-backed implementation of DataGenerator.
|
||||
*/
|
||||
|
||||
#include <cstdint>
|
||||
#include <filesystem>
|
||||
#include <memory>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
#include "../services/prompting/prompt_directory.h"
|
||||
#include "data_generation/data_generator.h"
|
||||
#include "data_generation/prompt_formatting/prompt_formatter.h"
|
||||
#include "data_model/models.h"
|
||||
#include "services/logging/logger.h"
|
||||
#include "services/prompting/prompt_directory.h"
|
||||
|
||||
struct llama_model;
|
||||
struct llama_context;
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
* inputs.
|
||||
*/
|
||||
|
||||
#include <boost/program_options.hpp>
|
||||
#include <cstdint>
|
||||
#include <filesystem>
|
||||
#include <memory>
|
||||
@@ -16,6 +15,8 @@
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
#include <boost/program_options.hpp>
|
||||
|
||||
class ILogger;
|
||||
|
||||
namespace prog_opts = boost::program_options;
|
||||
|
||||
@@ -9,10 +9,11 @@
|
||||
* readable output. Adapted from Boost JSON library examples.
|
||||
*/
|
||||
|
||||
#include <boost/json.hpp>
|
||||
#include <ostream>
|
||||
#include <string>
|
||||
|
||||
#include <boost/json.hpp>
|
||||
|
||||
/**
|
||||
* @brief Pretty-prints a JSON value to an output stream with indentation.
|
||||
*
|
||||
@@ -106,4 +107,4 @@ inline void PrettyPrint(std::ostream& outstream,
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_PRETTY_PRINT_H_
|
||||
|
||||
@@ -6,13 +6,13 @@
|
||||
* @brief Declarations for connection-level SQLite helper functions.
|
||||
*/
|
||||
|
||||
#include <sqlite3.h>
|
||||
|
||||
#include <filesystem>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
#include "sqlite_handle_types.h"
|
||||
#include <sqlite3.h>
|
||||
|
||||
#include "services/database/sqlite_handle_types.h"
|
||||
|
||||
namespace sqlite_export_service_internal {
|
||||
|
||||
|
||||
@@ -11,10 +11,10 @@
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
|
||||
#include "../datetime/date_time_provider.h"
|
||||
#include "data_model/models.h"
|
||||
#include "export_service.h"
|
||||
#include "sqlite_export_service_helpers.h"
|
||||
#include "services/database/export_service.h"
|
||||
#include "services/database/sqlite_export_service_helpers.h"
|
||||
#include "services/datetime/date_time_provider.h"
|
||||
|
||||
/**
|
||||
* @brief Persists generated brewery records into a fresh SQLite database.
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
|
||||
/* Umbrella header for backward compatibility. */
|
||||
|
||||
#include "sqlite_connection_helpers.h"
|
||||
#include "sqlite_handle_types.h"
|
||||
#include "sqlite_statement_helpers.h"
|
||||
#include "services/database/sqlite_connection_helpers.h"
|
||||
#include "services/database/sqlite_handle_types.h"
|
||||
#include "services/database/sqlite_statement_helpers.h"
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATABASE_SQLITE_EXPORT_SERVICE_HELPERS_H_
|
||||
|
||||
@@ -5,11 +5,11 @@
|
||||
* Shared handle and parameter type declarations used by SQLite helper units.
|
||||
*/
|
||||
|
||||
#include <sqlite3.h>
|
||||
|
||||
#include <memory>
|
||||
#include <string_view>
|
||||
|
||||
#include <sqlite3.h>
|
||||
|
||||
namespace sqlite_export_service_internal {
|
||||
|
||||
struct SqliteDatabaseDeleter {
|
||||
|
||||
@@ -7,13 +7,13 @@
|
||||
* constants.
|
||||
*/
|
||||
|
||||
#include <sqlite3.h>
|
||||
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
#include "sqlite_handle_types.h"
|
||||
#include <sqlite3.h>
|
||||
|
||||
#include "services/database/sqlite_handle_types.h"
|
||||
|
||||
namespace sqlite_export_service_internal {
|
||||
|
||||
@@ -109,30 +109,36 @@ INSERT INTO users (
|
||||
|
||||
// sqlite3_bind_*() parameter indices are 1-based, matching the "?"
|
||||
// placeholder order in the SQL above.
|
||||
inline constexpr int kLocationCityBindIndex = 1;
|
||||
inline constexpr int kLocationStateProvinceBindIndex = 2;
|
||||
inline constexpr int kLocationIso31662BindIndex = 3;
|
||||
inline constexpr int kLocationCountryBindIndex = 4;
|
||||
inline constexpr int kLocationIso31661BindIndex = 5;
|
||||
inline constexpr int kLocationLanguagesBindIndex = 6;
|
||||
inline constexpr int kLocationLatitudeBindIndex = 7;
|
||||
inline constexpr int kLocationLongitudeBindIndex = 8;
|
||||
enum eLocationBindIndex {
|
||||
kLocationCityBindIndex = 1,
|
||||
kLocationStateProvinceBindIndex,
|
||||
kLocationIso31662BindIndex,
|
||||
kLocationCountryBindIndex,
|
||||
kLocationIso31661BindIndex,
|
||||
kLocationLanguagesBindIndex,
|
||||
kLocationLatitudeBindIndex,
|
||||
kLocationLongitudeBindIndex,
|
||||
};
|
||||
|
||||
inline constexpr int kBreweryLocationIdBindIndex = 1;
|
||||
inline constexpr int kBreweryEnglishNameBindIndex = 2;
|
||||
inline constexpr int kBreweryEnglishDescriptionBindIndex = 3;
|
||||
inline constexpr int kBreweryLocalNameBindIndex = 4;
|
||||
inline constexpr int kBreweryLocalDescriptionBindIndex = 5;
|
||||
enum BreweryBindIndex {
|
||||
kBreweryLocationIdBindIndex = 1,
|
||||
kBreweryEnglishNameBindIndex,
|
||||
kBreweryEnglishDescriptionBindIndex,
|
||||
kBreweryLocalNameBindIndex,
|
||||
kBreweryLocalDescriptionBindIndex,
|
||||
};
|
||||
|
||||
inline constexpr int kUserLocationIdBindIndex = 1;
|
||||
inline constexpr int kUserFirstNameBindIndex = 2;
|
||||
inline constexpr int kUserLastNameBindIndex = 3;
|
||||
inline constexpr int kUserGenderBindIndex = 4;
|
||||
inline constexpr int kUserUsernameBindIndex = 5;
|
||||
inline constexpr int kUserBioBindIndex = 6;
|
||||
inline constexpr int kUserActivityWeightBindIndex = 7;
|
||||
inline constexpr int kUserEmailBindIndex = 8;
|
||||
inline constexpr int kUserDateOfBirthBindIndex = 9;
|
||||
enum UserBindIndex {
|
||||
kUserLocationIdBindIndex = 1,
|
||||
kUserFirstNameBindIndex,
|
||||
kUserLastNameBindIndex,
|
||||
kUserGenderBindIndex,
|
||||
kUserUsernameBindIndex,
|
||||
kUserBioBindIndex,
|
||||
kUserActivityWeightBindIndex,
|
||||
kUserEmailBindIndex,
|
||||
kUserDateOfBirthBindIndex,
|
||||
};
|
||||
|
||||
SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle,
|
||||
std::string_view sql,
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "enrichment_service.h"
|
||||
#include "services/enrichment/enrichment_service.h"
|
||||
|
||||
/**
|
||||
* @brief Enrichment service that returns no context for any location.
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
#include <string_view>
|
||||
#include <unordered_map>
|
||||
|
||||
#include "enrichment_service.h"
|
||||
#include "services/enrichment/enrichment_service.h"
|
||||
#include "services/logging/logger.h"
|
||||
#include "web_client/web_client.h"
|
||||
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
* a bounded channel for later processing by the dispatcher.
|
||||
*/
|
||||
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_
|
||||
|
||||
#include <string_view>
|
||||
|
||||
@@ -50,4 +50,4 @@ class LogProducer final : public ILogger {
|
||||
BoundedChannel<LogEntry>& channel_;
|
||||
};
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_
|
||||
|
||||
@@ -52,4 +52,4 @@ class HttpWebClient final : public WebClient {
|
||||
std::shared_ptr<ILogger> logger_;
|
||||
};
|
||||
|
||||
#endif
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_WEB_CLIENT_HTTP_WEB_CLIENT_H_
|
||||
|
||||
@@ -14,10 +14,9 @@ BiergartenPipelineOrchestrator::BiergartenPipelineOrchestrator(
|
||||
std::unique_ptr<IExportService> exporter,
|
||||
std::unique_ptr<ICuratedDataService> curated_data_service,
|
||||
const ApplicationOptions& application_options)
|
||||
: logger_(std::move(logger)),
|
||||
context_service_(std::move(context_service)),
|
||||
generator_(std::move(generator)),
|
||||
exporter_(std::move(exporter)),
|
||||
curated_data_service_(std::move(curated_data_service)),
|
||||
application_options_(application_options) {
|
||||
}
|
||||
: logger_(std::move(logger)),
|
||||
context_service_(std::move(context_service)),
|
||||
generator_(std::move(generator)),
|
||||
exporter_(std::move(exporter)),
|
||||
curated_data_service_(std::move(curated_data_service)),
|
||||
application_options_(application_options) {}
|
||||
@@ -24,7 +24,7 @@ std::string Sanitize(std::string_view value) {
|
||||
for (const char character : value) {
|
||||
if (std::isalnum(static_cast<unsigned char>(character)) != 0) {
|
||||
out.push_back(static_cast<char>(
|
||||
std::tolower(static_cast<unsigned char>(character))));
|
||||
std::tolower(static_cast<unsigned char>(character))));
|
||||
}
|
||||
}
|
||||
return out;
|
||||
@@ -69,7 +69,7 @@ std::string GenerateDateOfBirth(std::mt19937& rng) {
|
||||
static_cast<unsigned>(birth_ymd.month()),
|
||||
static_cast<unsigned>(birth_ymd.day()));
|
||||
}
|
||||
} // namespace
|
||||
} // namespace
|
||||
|
||||
void BiergartenPipelineOrchestrator::GenerateUsers(
|
||||
std::span<const EnrichedCity> cities) {
|
||||
@@ -86,8 +86,8 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
|
||||
}
|
||||
|
||||
const NamesByCountry names_by_country =
|
||||
curated_data_service_->LoadNamesByCountry(
|
||||
"forenames-by-country.json", "surnames-by-country.json");
|
||||
curated_data_service_->LoadNamesByCountry("forenames-by-country.json",
|
||||
"surnames-by-country.json");
|
||||
|
||||
std::mt19937 rng(std::random_device{}());
|
||||
std::uniform_int_distribution<size_t> persona_dist(0, personas.size() - 1);
|
||||
@@ -98,8 +98,8 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
|
||||
|
||||
const auto generate_record =
|
||||
[this, &rng, &skipped_count](
|
||||
const EnrichedCity& city, const UserPersona& persona,
|
||||
const Name& sampled_name) -> std::optional<UserRecord> {
|
||||
const EnrichedCity& city, const UserPersona& persona,
|
||||
const Name& sampled_name) -> std::optional<UserRecord> {
|
||||
try {
|
||||
std::unordered_set<std::string> used_email_local_parts;
|
||||
|
||||
@@ -125,19 +125,19 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
|
||||
};
|
||||
|
||||
const auto export_record = [this,
|
||||
&export_failed_count](const UserRecord& record) {
|
||||
&export_failed_count](const UserRecord& record) {
|
||||
try {
|
||||
exporter_->ProcessRecord(record);
|
||||
} catch (const std::exception& export_exception) {
|
||||
++export_failed_count;
|
||||
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::UserGeneration,
|
||||
.message = std::format("[Pipeline] Generated user for '{}' ({}) but "
|
||||
"SQLite export failed: {}",
|
||||
record.location.city, record.location.country,
|
||||
export_exception.what())});
|
||||
{.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::UserGeneration,
|
||||
.message = std::format("[Pipeline] Generated user for '{}' ({}) but "
|
||||
"SQLite export failed: {}",
|
||||
record.location.city, record.location.country,
|
||||
export_exception.what())});
|
||||
}
|
||||
};
|
||||
|
||||
@@ -171,11 +171,11 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
|
||||
|
||||
if (skipped_count > 0) {
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::UserGeneration,
|
||||
.message = std::format(
|
||||
"[Pipeline] Skipped {} city/cities during user generation",
|
||||
skipped_count)});
|
||||
{.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::UserGeneration,
|
||||
.message = std::format(
|
||||
"[Pipeline] Skipped {} city/cities during user generation",
|
||||
skipped_count)});
|
||||
}
|
||||
|
||||
if (export_failed_count > 0) {
|
||||
|
||||
@@ -3,12 +3,13 @@
|
||||
* @brief BiergartenPipelineOrchestrator::LogResults() implementation.
|
||||
*/
|
||||
|
||||
#include <boost/json/array.hpp>
|
||||
#include <chrono>
|
||||
#include <format>
|
||||
|
||||
#include "../../includes/json_handling/pretty_print.h"
|
||||
#include <boost/json/array.hpp>
|
||||
|
||||
#include "biergarten_pipeline_orchestrator.h"
|
||||
#include "json_handling/pretty_print.h"
|
||||
#include "services/logging/logger.h"
|
||||
|
||||
void BiergartenPipelineOrchestrator::LogResults() const {
|
||||
|
||||
@@ -6,7 +6,6 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <boost/json.hpp>
|
||||
#include <cctype>
|
||||
#include <optional>
|
||||
#include <span>
|
||||
@@ -15,8 +14,11 @@
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
#include <boost/json.hpp>
|
||||
#include <llama.h>
|
||||
|
||||
#include "data_generation/llama_generator_helpers.h"
|
||||
#include "llama.h"
|
||||
|
||||
namespace {
|
||||
/**
|
||||
* String trimming: removes leading and trailing whitespace
|
||||
|
||||
@@ -14,9 +14,10 @@
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
#include <llama.h>
|
||||
|
||||
#include "data_generation/llama_generator.h"
|
||||
#include "data_generation/llama_generator_helpers.h"
|
||||
#include "llama.h"
|
||||
|
||||
static constexpr size_t kPromptTokenSlack = 8;
|
||||
// Minimum tokens to keep when using top-p sampling. Ensures at least one
|
||||
|
||||
@@ -11,8 +11,9 @@
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
|
||||
#include <llama.h>
|
||||
|
||||
#include "data_model/models.h"
|
||||
#include "llama.h"
|
||||
|
||||
static constexpr uint32_t kMaxContextSize = 32768U;
|
||||
|
||||
|
||||
@@ -10,9 +10,10 @@
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
#include <ggml-backend.h>
|
||||
#include <llama.h>
|
||||
|
||||
#include "data_generation/llama_generator.h"
|
||||
#include "ggml-backend.h"
|
||||
#include "llama.h"
|
||||
|
||||
// Maximum batch size for decode operations. Capping the batch prevents
|
||||
// excessive memory allocation while maintaining inference performance.
|
||||
|
||||
@@ -6,7 +6,6 @@
|
||||
|
||||
#include "json_handling/json_loader.h"
|
||||
|
||||
#include <boost/json.hpp>
|
||||
#include <format>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
@@ -17,6 +16,8 @@
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
|
||||
#include <boost/json.hpp>
|
||||
|
||||
#include "services/logging/logger.h"
|
||||
|
||||
static std::string ReadRequiredString(const boost::json::object& object,
|
||||
@@ -137,7 +138,6 @@ std::vector<Location> JsonLoader::LoadLocations(
|
||||
.longitude = ReadRequiredNumber(object, "longitude"),
|
||||
});
|
||||
}
|
||||
|
||||
return locations;
|
||||
}
|
||||
|
||||
|
||||
@@ -4,11 +4,6 @@
|
||||
* initializes shared infrastructure, and executes the pipeline entry flow.
|
||||
*/
|
||||
|
||||
#include <spdlog/fmt/fmt.h>
|
||||
#include <spdlog/spdlog.h>
|
||||
|
||||
#include <boost/di.hpp>
|
||||
#include <boost/program_options.hpp>
|
||||
#include <chrono>
|
||||
#include <exception>
|
||||
#include <format>
|
||||
@@ -18,27 +13,12 @@
|
||||
#include <string>
|
||||
#include <thread>
|
||||
|
||||
#include "biergarten_pipeline_orchestrator.h"
|
||||
#include "concurrency/bounded_channel.h"
|
||||
#include "data_generation/llama_generator.h"
|
||||
#include "data_generation/mock_generator.h"
|
||||
#include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h"
|
||||
#include "data_model/models.h"
|
||||
#include "json_handling/json_loader.h"
|
||||
#include "llama_backend_state.h"
|
||||
#include "services/curated_data/curated_data_service.h"
|
||||
#include "services/database/export_service.h"
|
||||
#include "services/database/sqlite_export_service.h"
|
||||
#include "services/datetime/timer.h"
|
||||
#include "services/enrichment/enrichment_service.h"
|
||||
#include "services/enrichment/mock_enrichment.h"
|
||||
#include "services/enrichment/wikipedia_service.h"
|
||||
#include "services/logging/log_dispatcher.h"
|
||||
#include "services/logging/log_entry.h"
|
||||
#include "services/logging/log_producer.h"
|
||||
#include "services/logging/logger.h"
|
||||
#include "services/prompting/prompt_directory.h"
|
||||
#include "web_client/http_web_client.h"
|
||||
#include <boost/di.hpp>
|
||||
#include <boost/program_options.hpp>
|
||||
#include <spdlog/fmt/fmt.h>
|
||||
#include <spdlog/spdlog.h>
|
||||
|
||||
#include "biergarten_pipeline.h"
|
||||
|
||||
namespace di = boost::di;
|
||||
|
||||
@@ -49,7 +29,7 @@ int main(const int argc, char** argv) {
|
||||
spdlog::set_pattern("│ %Y-%m-%d %H:%M:%S.%e │ %^%-7l%$ │ %v");
|
||||
BoundedChannel<LogEntry> log_channel(kLogMaxCount);
|
||||
|
||||
auto log_dispatcher = //
|
||||
auto log_dispatcher = //
|
||||
std::make_unique<LogDispatcher>(log_channel);
|
||||
std::shared_ptr<ILogger> log_producer =
|
||||
std::make_shared<LogProducer>(log_channel);
|
||||
@@ -111,17 +91,17 @@ int main(const int argc, char** argv) {
|
||||
if (options.generator.use_mocked) {
|
||||
{
|
||||
log_producer->Log(
|
||||
{.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::Startup,
|
||||
.message = "Prompt formatter: none (mock mode)"});
|
||||
{.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::Startup,
|
||||
.message = "Prompt formatter: none (mock mode)"});
|
||||
}
|
||||
return std::unique_ptr<IPromptFormatter>(nullptr);
|
||||
}
|
||||
{
|
||||
log_producer->Log(
|
||||
{.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::Startup,
|
||||
.message = "Prompt formatter: Gemma4JinjaPromptFormatter"});
|
||||
{.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::Startup,
|
||||
.message = "Prompt formatter: Gemma4JinjaPromptFormatter"});
|
||||
}
|
||||
return std::unique_ptr<IPromptFormatter>(
|
||||
std::make_unique<Gemma4JinjaPromptFormatter>());
|
||||
@@ -145,7 +125,7 @@ int main(const int argc, char** argv) {
|
||||
}),
|
||||
di::bind<IEnrichmentService>().to(
|
||||
[options, &log_producer](
|
||||
const auto& inj) -> std::unique_ptr<IEnrichmentService> {
|
||||
const auto& inj) -> std::unique_ptr<IEnrichmentService> {
|
||||
if (options.generator.use_mocked) {
|
||||
log_producer->Log({.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::Startup,
|
||||
@@ -164,8 +144,7 @@ int main(const int argc, char** argv) {
|
||||
}),
|
||||
di::bind<DataGenerator>().to(
|
||||
[&options, &model_path, &sampling, &prompt_directory,
|
||||
&log_producer
|
||||
](const auto& inj) -> std::unique_ptr<DataGenerator> {
|
||||
&log_producer](const auto& inj) -> std::unique_ptr<DataGenerator> {
|
||||
if (options.generator.use_mocked) {
|
||||
log_producer->Log({.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::Startup,
|
||||
@@ -175,13 +154,13 @@ int main(const int argc, char** argv) {
|
||||
}
|
||||
|
||||
log_producer->Log(
|
||||
{.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::Startup,
|
||||
.message = std::format(
|
||||
"Generator: LlamaGenerator | model={} | "
|
||||
"temp={:.2f} top_p={:.2f} top_k={} n_ctx={} seed={}",
|
||||
model_path, sampling.temperature, sampling.top_p,
|
||||
sampling.top_k, sampling.n_ctx, sampling.seed)});
|
||||
{.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::Startup,
|
||||
.message = std::format(
|
||||
"Generator: LlamaGenerator | model={} | "
|
||||
"temp={:.2f} top_p={:.2f} top_k={} n_ctx={} seed={}",
|
||||
model_path, sampling.temperature, sampling.top_p,
|
||||
sampling.top_k, sampling.n_ctx, sampling.seed)});
|
||||
|
||||
return std::make_unique<LlamaGenerator>(
|
||||
options, model_path, log_producer,
|
||||
|
||||
@@ -3,13 +3,14 @@
|
||||
* @brief WikipediaEnrichmentService::FetchExtract() implementation.
|
||||
*/
|
||||
|
||||
#include <boost/json.hpp>
|
||||
#include <chrono>
|
||||
#include <format>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <thread>
|
||||
|
||||
#include <boost/json.hpp>
|
||||
|
||||
#include "services/enrichment/wikipedia_service.h"
|
||||
|
||||
using namespace boost;
|
||||
|
||||
@@ -6,10 +6,10 @@
|
||||
*/
|
||||
#include "services/logging/log_dispatcher.h"
|
||||
|
||||
#include <spdlog/spdlog.h>
|
||||
|
||||
#include <string>
|
||||
|
||||
#include <spdlog/spdlog.h>
|
||||
|
||||
#include "concurrency/bounded_channel.h"
|
||||
#include "services/logging/log_entry.h"
|
||||
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
#include "services/database/sqlite_statement_helpers.h"
|
||||
|
||||
#include <boost/json.hpp>
|
||||
#include <cstring>
|
||||
#include <limits>
|
||||
#include <memory>
|
||||
#include <stdexcept>
|
||||
|
||||
#include <boost/json.hpp>
|
||||
|
||||
#include "services/database/sqlite_connection_helpers.h"
|
||||
|
||||
namespace sqlite_export_service_internal {
|
||||
|
||||
@@ -5,8 +5,6 @@
|
||||
|
||||
#include "web_client/http_web_client.h"
|
||||
|
||||
#include <httplib.h>
|
||||
|
||||
#include <chrono>
|
||||
#include <format>
|
||||
#include <regex>
|
||||
@@ -14,6 +12,8 @@
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
#include <httplib.h>
|
||||
|
||||
#include "services/logging/logger.h"
|
||||
|
||||
namespace {
|
||||
|
||||
Reference in New Issue
Block a user