mirror of
https://github.com/aaronpo97/the-biergarten-app.git
synced 2026-05-31 17:53:59 +00:00
Implement BoundedChannel and multithreaded logging infra
This commit is contained in:
@@ -155,13 +155,13 @@ target_sources(${PROJECT_NAME} PRIVATE
|
||||
src/application_options/parse_arguments.cc
|
||||
)
|
||||
|
||||
# --- biergarten_data_generator ---
|
||||
# --- biergarten_pipeline_orchestrator ---
|
||||
target_sources(${PROJECT_NAME} PRIVATE
|
||||
src/biergarten_data_generator/log_results.cc
|
||||
src/biergarten_data_generator/biergarten_data_generator.cc
|
||||
src/biergarten_data_generator/generate_breweries.cc
|
||||
src/biergarten_data_generator/run.cc
|
||||
src/biergarten_data_generator/query_cities_with_countries.cc
|
||||
src/biergarten_pipeline_orchestrator/log_results.cc
|
||||
src/biergarten_pipeline_orchestrator/biergarten_data_generator.cc
|
||||
src/biergarten_pipeline_orchestrator/generate_breweries.cc
|
||||
src/biergarten_pipeline_orchestrator/run.cc
|
||||
src/biergarten_pipeline_orchestrator/query_cities_with_countries.cc
|
||||
)
|
||||
|
||||
# --- web_client ---
|
||||
@@ -202,17 +202,23 @@ target_sources(${PROJECT_NAME} PRIVATE
|
||||
|
||||
# --- services: sqlite ---
|
||||
target_sources(${PROJECT_NAME} PRIVATE
|
||||
src/services/sqlite/process_record.cc
|
||||
src/services/sqlite/sqlite_export_service.cc
|
||||
src/services/sqlite/finalize.cc
|
||||
src/services/sqlite/initialize.cc
|
||||
src/services/sqlite/helpers/sqlite_connection_helpers.cc
|
||||
src/services/sqlite/helpers/sqlite_statement_helpers.cc
|
||||
src/services/sqlite/process_record.cc
|
||||
src/services/sqlite/sqlite_export_service.cc
|
||||
src/services/sqlite/finalize.cc
|
||||
src/services/sqlite/initialize.cc
|
||||
src/services/sqlite/helpers/sqlite_connection_helpers.cc
|
||||
src/services/sqlite/helpers/sqlite_statement_helpers.cc
|
||||
)
|
||||
|
||||
# --- services: logging ---
|
||||
target_sources(${PROJECT_NAME} PRIVATE
|
||||
"src/services/logging/channel_logger.cc"
|
||||
src/services/logging/log_consumer.cc
|
||||
)
|
||||
|
||||
# --- services (top-level) ---
|
||||
target_sources(${PROJECT_NAME} PRIVATE
|
||||
src/services/prompt_directory.cc
|
||||
src/services/prompt_directory.cc
|
||||
)
|
||||
|
||||
# 6. Include Directories, Link Libraries & Compile Definitions
|
||||
|
||||
@@ -3,7 +3,10 @@
|
||||
|
||||
/**
|
||||
* @file biergarten_data_generator.h
|
||||
* @brief Core orchestration class for pipeline data generation.
|
||||
* @brief Orchestration for end-to-end brewery data generation pipeline.
|
||||
*
|
||||
* Intent: Coordinates location loading, enrichment, and generation phases
|
||||
* to produce a complete dataset. Coordinates dependencies via composition root.
|
||||
*/
|
||||
|
||||
#include <memory>
|
||||
@@ -21,19 +24,21 @@
|
||||
* This class encapsulates the core logic for generating brewery data.
|
||||
* It handles location loading, city enrichment, and brewery generation.
|
||||
*/
|
||||
class BiergartenDataGenerator {
|
||||
class BiergartenPipelineOrchestrator {
|
||||
public:
|
||||
/**
|
||||
* @brief Construct a BiergartenDataGenerator with injected dependencies.
|
||||
*
|
||||
* @param context_service Context provider for sampled locations.
|
||||
* @param generator Brewery and user data generator.
|
||||
* @param exporter Storage backend for generated brewery data.
|
||||
*/
|
||||
BiergartenDataGenerator(std::unique_ptr<IEnrichmentService> context_service,
|
||||
std::unique_ptr<DataGenerator> generator,
|
||||
std::unique_ptr<IExportService> exporter,
|
||||
const ApplicationOptions& application_options);
|
||||
/**
|
||||
* @brief Constructs the orchestrator with injected pipeline dependencies.
|
||||
*
|
||||
* @param context_service Provides regional context for locations.
|
||||
* @param generator Implementation (Llama or Mock) for brewery/user generation.
|
||||
* @param exporter Database backend for persisting generated records.
|
||||
* @param application_options CLI configuration and paths.
|
||||
*/
|
||||
BiergartenPipelineOrchestrator(
|
||||
std::unique_ptr<IEnrichmentService> context_service,
|
||||
std::unique_ptr<DataGenerator> generator,
|
||||
std::unique_ptr<IExportService> exporter,
|
||||
const ApplicationOptions& application_options);
|
||||
|
||||
/**
|
||||
* @brief Run the data generation pipeline.
|
||||
@@ -54,10 +59,11 @@ class BiergartenDataGenerator {
|
||||
/// @brief Generator dependency selected in the composition root.
|
||||
std::unique_ptr<DataGenerator> generator_;
|
||||
|
||||
/// @brief Storage backend for generated brewery records.
|
||||
std::unique_ptr<IExportService> exporter_;
|
||||
/// @brief Storage backend for generated brewery records.
|
||||
std::unique_ptr<IExportService> exporter_;
|
||||
|
||||
const ApplicationOptions application_options_;
|
||||
/// @brief CLI configuration: paths, model settings, generation parameters.
|
||||
ApplicationOptions application_options_;
|
||||
|
||||
/**
|
||||
* @brief Load locations from JSON and sample cities.
|
||||
|
||||
73
tooling/pipeline/includes/concurrency/bounded_channel.h
Normal file
73
tooling/pipeline/includes/concurrency/bounded_channel.h
Normal file
@@ -0,0 +1,73 @@
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_CONCURRENCY_BOUNDED_CHANNEL_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_CONCURRENCY_BOUNDED_CHANNEL_H_
|
||||
|
||||
#include <condition_variable>
|
||||
#include <cstddef>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <queue>
|
||||
|
||||
/**
|
||||
* @file bounded_channel.h
|
||||
* @brief Thread-safe, bounded multi-producer/multi-consumer synchronous channel.
|
||||
*
|
||||
* Intent: Enables asynchronous inter-thread communication with backpressure.
|
||||
* Models a synchronous channel where producers/consumers block on capacity limits.
|
||||
*/
|
||||
|
||||
/**
|
||||
* @class BoundedChannel
|
||||
* @brief MPMC channel with fixed capacity and blocking semantics.
|
||||
*
|
||||
* Producers block when buffer is full; consumers block when empty.
|
||||
* Close() unblocks all waiters and signals channel exhaustion.
|
||||
*/
|
||||
template <typename T>
|
||||
class BoundedChannel {
|
||||
// -------------------------------------------------------------------------
|
||||
// Internal state — all access must be guarded by mutex_.
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
std::queue<T> queue_;
|
||||
|
||||
std::mutex mutex_;
|
||||
|
||||
std::condition_variable not_full_;
|
||||
|
||||
std::condition_variable not_empty_;
|
||||
|
||||
std::size_t capacity_;
|
||||
|
||||
bool closed_ = false;
|
||||
|
||||
public:
|
||||
/**
|
||||
* @brief Construct a bounded channel with the given capacity.
|
||||
* @param capacity Maximum number of items the channel may hold.
|
||||
*/
|
||||
explicit BoundedChannel(std::size_t capacity) : capacity_(capacity) {}
|
||||
|
||||
/**
|
||||
* @brief Send an item into the channel. Blocks when the channel is full.
|
||||
* @param item Move-only item to enqueue.
|
||||
*/
|
||||
void Send(T item);
|
||||
|
||||
/**
|
||||
* @brief Receive an item from the channel. Blocks when the channel is
|
||||
* empty.
|
||||
* @return std::optional<T> containing the item, or std::nullopt when the
|
||||
* channel is closed and drained.
|
||||
*/
|
||||
std::optional<T> Receive();
|
||||
|
||||
/**
|
||||
* @brief Close the channel and unblock all waiting threads. Idempotent.
|
||||
*/
|
||||
void Close();
|
||||
};
|
||||
|
||||
// Include the template implementation
|
||||
#include "bounded_channel.tcc"
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_CONCURRENCY_BOUNDED_CHANNEL_H_
|
||||
57
tooling/pipeline/includes/concurrency/bounded_channel.tcc
Normal file
57
tooling/pipeline/includes/concurrency/bounded_channel.tcc
Normal file
@@ -0,0 +1,57 @@
|
||||
#include "bounded_channel.h"
|
||||
|
||||
template <typename T>
|
||||
void BoundedChannel<T>::Send(T item) {
|
||||
// Acquire exclusive ownership of the mutex; released automatically on scope exit.
|
||||
std::unique_lock lock(mutex_);
|
||||
|
||||
// Block until there is space in the queue or the channel has been closed.
|
||||
// The predicate guards against spurious wakeups.
|
||||
not_full_.wait(lock, [&] { return queue_.size() < capacity_ || closed_; });
|
||||
|
||||
// If the channel was closed while waiting, discard the item and return.
|
||||
if (closed_) return;
|
||||
|
||||
// Move the item into the queue to avoid an unnecessary copy.
|
||||
queue_.push(std::move(item));
|
||||
|
||||
// Wake one blocked Receive() call to signal that data is now available.
|
||||
not_empty_.notify_one();
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
std::optional<T> BoundedChannel<T>::Receive() {
|
||||
// Acquire exclusive ownership of the mutex.
|
||||
std::unique_lock lock(mutex_);
|
||||
|
||||
// Block until the queue is non-empty or the channel has been closed.
|
||||
// The predicate guards against spurious wakeups.
|
||||
not_empty_.wait(lock, [&] { return !queue_.empty() || closed_; });
|
||||
|
||||
// If woken due to closure and no items remain, signal exhaustion via nullopt.
|
||||
if (queue_.empty()) return std::nullopt;
|
||||
|
||||
// Move the front item out of the queue to avoid an unnecessary copy.
|
||||
T item = std::move(queue_.front());
|
||||
queue_.pop();
|
||||
|
||||
// Wake one blocked Send() call to signal that a slot has opened.
|
||||
not_full_.notify_one();
|
||||
|
||||
return item;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void BoundedChannel<T>::Close() {
|
||||
// Acquire exclusive ownership of the mutex to ensure visibility of the flag.
|
||||
std::unique_lock lock(mutex_);
|
||||
|
||||
// Mark the channel as closed; subsequent Send() calls will be dropped.
|
||||
closed_ = true;
|
||||
|
||||
// Wake all blocked Send() callers so they can observe the closed flag and exit.
|
||||
not_full_.notify_all();
|
||||
|
||||
// Wake all blocked Receive() callers so they can drain remaining items or return nullopt.
|
||||
not_empty_.notify_all();
|
||||
}
|
||||
54
tooling/pipeline/includes/services/logging/channel_logger.h
Normal file
54
tooling/pipeline/includes/services/logging/channel_logger.h
Normal file
@@ -0,0 +1,54 @@
|
||||
/**
|
||||
* @file services/logging/channel_logger.h
|
||||
* @brief Channel-backed producer for asynchronous pipeline logging.
|
||||
*
|
||||
* Intent: Decouple logging from synchronous I/O by forwarding entries to a
|
||||
* bounded channel. LogConsumer drains the channel on a dedicated thread.
|
||||
*/
|
||||
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_
|
||||
|
||||
#include <string_view>
|
||||
|
||||
#include "concurrency/bounded_channel.h"
|
||||
#include "services/logging/log_entry.h"
|
||||
#include "services/logging/logger.h"
|
||||
|
||||
/**
|
||||
* @class ChannelLogger
|
||||
* @brief ILogger implementation that sends entries to a BoundedChannel.
|
||||
*
|
||||
* Non-copyable, non-movable. Holds a non-owning reference to the channel.
|
||||
*/
|
||||
class ChannelLogger final : public ILogger {
|
||||
public:
|
||||
/**
|
||||
* @brief Construct a channel-backed logger.
|
||||
*
|
||||
* @param channel Reference to bounded channel for log entry transfer.
|
||||
* Channel must outlive this logger instance.
|
||||
*/
|
||||
explicit ChannelLogger(BoundedChannel<LogEntry>& channel);
|
||||
|
||||
ChannelLogger(const ChannelLogger&) = delete;
|
||||
ChannelLogger& operator=(const ChannelLogger&) = delete;
|
||||
ChannelLogger(ChannelLogger&&) = delete;
|
||||
ChannelLogger& operator=(ChannelLogger&&) = delete;
|
||||
|
||||
~ChannelLogger() override = default;
|
||||
|
||||
/**
|
||||
* @brief Queue a log entry for asynchronous processing.
|
||||
*
|
||||
* Blocks if the channel is full (backpressure). Returns immediately
|
||||
* if the channel is closed.
|
||||
*/
|
||||
void Log(LogLevel level, PipelinePhase phase,
|
||||
std::string_view message) override;
|
||||
|
||||
private:
|
||||
BoundedChannel<LogEntry>& channel_;
|
||||
};
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_
|
||||
59
tooling/pipeline/includes/services/logging/log_consumer.h
Normal file
59
tooling/pipeline/includes/services/logging/log_consumer.h
Normal file
@@ -0,0 +1,59 @@
|
||||
/**
|
||||
* @file services/logging/log_consumer.h
|
||||
* @brief Dedicated log consumer/drain for asynchronous pipeline logging.
|
||||
*
|
||||
* Intent: Dequeue LogEntry values from a BoundedChannel on a dedicated thread
|
||||
* and forward them to spdlog for I/O and formatting. Decouples application
|
||||
* logic from logging latency.
|
||||
*/
|
||||
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_CONSUMER_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_CONSUMER_H_
|
||||
|
||||
#include <spdlog/spdlog.h>
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "concurrency/bounded_channel.h"
|
||||
#include "services/logging/log_entry.h"
|
||||
|
||||
/**
|
||||
* @class LogConsumer
|
||||
* @brief Consumes log entries from channel and forwards to spdlog.
|
||||
*
|
||||
* Non-copyable, non-movable. Designed to run on its own dedicated std::thread.
|
||||
* Drains the channel until closure, then exits cleanly.
|
||||
*/
|
||||
class LogConsumer {
|
||||
public:
|
||||
/**
|
||||
* @brief Construct a log consumer.
|
||||
*
|
||||
* @param channel Reference to bounded channel for log entry retrieval.
|
||||
* Channel must outlive this consumer instance.
|
||||
*/
|
||||
explicit LogConsumer(BoundedChannel<LogEntry>& channel);
|
||||
|
||||
LogConsumer(const LogConsumer&) = delete;
|
||||
LogConsumer& operator=(const LogConsumer&) = delete;
|
||||
LogConsumer(LogConsumer&&) = delete;
|
||||
LogConsumer& operator=(LogConsumer&&) = delete;
|
||||
|
||||
/**
|
||||
* @brief Main loop: drain channel and forward entries to spdlog.
|
||||
*
|
||||
* Intended to be called once on a dedicated thread. Returns when:
|
||||
* - Channel is closed AND all queued entries are drained.
|
||||
*
|
||||
* Thread-safe for use from multiple ChannelLogger instances on other threads.
|
||||
*/
|
||||
void Run();
|
||||
|
||||
private:
|
||||
BoundedChannel<LogEntry>& channel_;
|
||||
|
||||
static spdlog::level::level_enum ToSpdlogLevel(LogLevel level);
|
||||
static std::string ToString(PipelinePhase phase);
|
||||
};
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_CONSUMER_H_
|
||||
65
tooling/pipeline/includes/services/logging/log_entry.h
Normal file
65
tooling/pipeline/includes/services/logging/log_entry.h
Normal file
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* @file services/logging/log_entry.h
|
||||
* @brief POD log entry structure for asynchronous pipeline logging.
|
||||
*
|
||||
* Intent: Lightweight, move-safe data transfer between logging producer
|
||||
* (ChannelLogger) and consumer (LogConsumer) via BoundedChannel<LogEntry>.
|
||||
*/
|
||||
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_ENTRY_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_ENTRY_H_
|
||||
|
||||
#include <chrono>
|
||||
#include <string>
|
||||
|
||||
/**
|
||||
* @enum LogLevel
|
||||
* @brief Severity levels for log entries.
|
||||
*/
|
||||
enum class LogLevel {
|
||||
Debug, ///< Development/debugging information.
|
||||
Info, ///< General informational messages.
|
||||
Warn, ///< Warning conditions.
|
||||
Error, ///< Error conditions.
|
||||
};
|
||||
|
||||
/**
|
||||
* @enum PipelinePhase
|
||||
* @brief Execution phases for contextual logging.
|
||||
*
|
||||
* Used to tag log entries by their processing stage, enabling phase-specific
|
||||
* analysis and filtering of the execution timeline.
|
||||
*/
|
||||
enum class PipelinePhase {
|
||||
Startup, ///< Initialization and validation.
|
||||
UserGeneration, ///< User profile generation.
|
||||
BreweryAndBeerGeneration, ///< Brewery and beer data generation.
|
||||
CheckinGeneration, ///< Checkin (visit) record generation.
|
||||
RatingGeneration, ///< Rating and review generation.
|
||||
FollowGeneration, ///< Follow relationship generation.
|
||||
Teardown, ///< Finalization and cleanup.
|
||||
};
|
||||
|
||||
/**
|
||||
* @struct LogEntry
|
||||
* @brief Single log event for asynchronous processing.
|
||||
*
|
||||
* All fields are value types, allowing safe move semantics across
|
||||
* BoundedChannel without shared ownership or synchronization overhead.
|
||||
*/
|
||||
struct LogEntry {
|
||||
/// @brief Timestamp when entry was created.
|
||||
std::chrono::system_clock::time_point timestamp =
|
||||
std::chrono::system_clock::now();
|
||||
|
||||
/// @brief Severity level of this entry.
|
||||
LogLevel level;
|
||||
|
||||
/// @brief Pipeline phase when entry was logged.
|
||||
PipelinePhase phase;
|
||||
|
||||
/// @brief Log message text.
|
||||
std::string message;
|
||||
};
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_ENTRY_H_
|
||||
45
tooling/pipeline/includes/services/logging/logger.h
Normal file
45
tooling/pipeline/includes/services/logging/logger.h
Normal file
@@ -0,0 +1,45 @@
|
||||
/**
|
||||
* @file services/logging/logger.h
|
||||
* @brief Abstract logging interface for pipeline components.
|
||||
*
|
||||
* Intent: Decouple logging from channel/worker implementation details.
|
||||
* All pipeline components depend on ILogger, enabling swappable backends.
|
||||
*/
|
||||
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOGGER_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOGGER_H_
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
#include "services/logging/log_entry.h"
|
||||
|
||||
/**
|
||||
* @class ILogger
|
||||
* @brief Minimal interface for submitting log entries.
|
||||
*
|
||||
* Non-copyable and non-movable. Implementations are typically short-lived,
|
||||
* created and owned by the composition root.
|
||||
*/
|
||||
class ILogger {
|
||||
public:
|
||||
ILogger() = default;
|
||||
ILogger(const ILogger&) = delete;
|
||||
ILogger& operator=(const ILogger&) = delete;
|
||||
ILogger(ILogger&&) = delete;
|
||||
ILogger& operator=(ILogger&&) = delete;
|
||||
virtual ~ILogger() = default;
|
||||
|
||||
/**
|
||||
* @brief Submit a log entry to the logging subsystem.
|
||||
*
|
||||
* @param level Log level (Debug, Info, Warn, Error).
|
||||
* @param phase Pipeline execution phase (Startup, Generation, Teardown, etc.).
|
||||
* @param message Log message text.
|
||||
*/
|
||||
virtual void Log(LogLevel level, PipelinePhase phase,
|
||||
std::string_view message) = 0;
|
||||
};
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOGGER_H_
|
||||
@@ -1,5 +1,5 @@
|
||||
/**
|
||||
* @file biergarten_data_generator/biergarten_data_generator.cc
|
||||
* @file biergarten_pipeline_orchestrator/biergarten_pipeline_orchestrator.cc
|
||||
* @brief BiergartenDataGenerator constructor implementation.
|
||||
*/
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
#include <utility>
|
||||
|
||||
BiergartenDataGenerator::BiergartenDataGenerator(
|
||||
BiergartenPipelineOrchestrator::BiergartenPipelineOrchestrator(
|
||||
std::unique_ptr<IEnrichmentService> context_service,
|
||||
std::unique_ptr<DataGenerator> generator,
|
||||
std::unique_ptr<IExportService> exporter,
|
||||
@@ -1,5 +1,5 @@
|
||||
/**
|
||||
* @file biergarten_data_generator/generate_breweries.cc
|
||||
* @file biergarten_pipeline_orchestrator/generate_breweries.cc
|
||||
* @brief BiergartenDataGenerator::GenerateBreweries() implementation.
|
||||
*/
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
#include "biergarten_data_generator.h"
|
||||
|
||||
void BiergartenDataGenerator::GenerateBreweries(
|
||||
void BiergartenPipelineOrchestrator::GenerateBreweries(
|
||||
std::span<const EnrichedCity> cities) {
|
||||
spdlog::info("\n=== SAMPLE BREWERY GENERATION ===");
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/**
|
||||
* @file biergarten_data_generator/log_results.cc
|
||||
* @file biergarten_pipeline_orchestrator/log_results.cc
|
||||
* @brief BiergartenDataGenerator::LogResults() implementation.
|
||||
*/
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
#include "biergarten_data_generator.h"
|
||||
|
||||
void BiergartenDataGenerator::LogResults() const {
|
||||
void BiergartenPipelineOrchestrator::LogResults() const {
|
||||
spdlog::info("\n=== GENERATED DATA DUMP ===");
|
||||
size_t index = 1;
|
||||
for (const auto& [location, brewery] : generated_breweries_) {
|
||||
@@ -1,5 +1,5 @@
|
||||
/**
|
||||
* @file biergarten_data_generator/query_cities_with_countries.cc
|
||||
* @file biergarten_pipeline_orchestrator/query_cities_with_countries.cc
|
||||
* @brief BiergartenDataGenerator::QueryCitiesWithCountries() implementation.
|
||||
*/
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
#include "biergarten_data_generator.h"
|
||||
#include "json_handling/json_loader.h"
|
||||
|
||||
std::vector<Location> BiergartenDataGenerator::QueryCitiesWithCountries() {
|
||||
std::vector<Location> BiergartenPipelineOrchestrator::QueryCitiesWithCountries() {
|
||||
spdlog::info("\n=== GEOGRAPHIC DATA OVERVIEW ===");
|
||||
|
||||
const std::filesystem::path locations_path = "locations.json";
|
||||
@@ -1,5 +1,5 @@
|
||||
/**
|
||||
* @file biergarten_data_generator/run.cc
|
||||
* @file biergarten_pipeline_orchestrator/run.cc
|
||||
* @brief BiergartenDataGenerator::Run() implementation.
|
||||
*/
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
|
||||
#include "biergarten_data_generator.h"
|
||||
|
||||
bool BiergartenDataGenerator::Run() {
|
||||
bool BiergartenPipelineOrchestrator::Run() {
|
||||
try {
|
||||
exporter_->Initialize();
|
||||
|
||||
@@ -12,8 +12,10 @@
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
|
||||
#include "biergarten_data_generator.h"
|
||||
#include "concurrency/bounded_channel.h"
|
||||
#include "data_generation/llama_generator.h"
|
||||
#include "data_generation/mock_generator.h"
|
||||
#include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h"
|
||||
@@ -25,12 +27,22 @@
|
||||
#include "services/enrichment/enrichment_service.h"
|
||||
#include "services/enrichment/mock_enrichment.h"
|
||||
#include "services/enrichment/wikipedia_service.h"
|
||||
#include "services/logging/channel_logger.h"
|
||||
#include "services/logging/log_consumer.h"
|
||||
#include "services/logging/log_entry.h"
|
||||
#include "services/logging/logger.h"
|
||||
#include "services/prompting/prompt_directory.h"
|
||||
#include "web_client/http_web_client.h"
|
||||
|
||||
namespace di = boost::di;
|
||||
|
||||
static constexpr size_t kLogMaxCount = 512;
|
||||
int main(const int argc, char** argv) {
|
||||
auto log_channel = std::make_shared<BoundedChannel<LogEntry>>(kLogMaxCount);
|
||||
ChannelLogger channel_logger(*log_channel);
|
||||
LogConsumer log_worker(*log_channel);
|
||||
std::thread log_thread([&log_worker] { log_worker.Run(); });
|
||||
|
||||
try {
|
||||
Timer timer;
|
||||
spdlog::set_pattern("[%Y-%m-%d %H:%M:%S.%e] [%^%l%$] %v");
|
||||
@@ -46,6 +58,8 @@ int main(const int argc, char** argv) {
|
||||
ParseArguments(argc, argv);
|
||||
|
||||
if (!parsed_options.has_value()) {
|
||||
log_channel->Close();
|
||||
log_thread.join();
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -54,66 +68,97 @@ int main(const int argc, char** argv) {
|
||||
const auto sampling =
|
||||
options.generator.sampling.value_or(SamplingOptions{});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Prompt directory
|
||||
// Conditionally constructed before the injector; moved into LlamaGenerator.
|
||||
// -----------------------------------------------------------------------
|
||||
std::unique_ptr<IPromptDirectory> prompt_directory;
|
||||
if (!options.generator.use_mocked) {
|
||||
try {
|
||||
prompt_directory =
|
||||
std::make_unique<PromptDirectory>(options.pipeline.prompt_dir);
|
||||
} catch (const std::exception& dir_error) {
|
||||
spdlog::error("[Startup] Invalid --prompt-dir: {}", dir_error.what());
|
||||
channel_logger.Log(
|
||||
LogLevel::Error, PipelinePhase::Startup,
|
||||
std::string("Invalid --prompt-dir: ") + dir_error.what());
|
||||
log_channel->Close();
|
||||
log_thread.join();
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Dependency injection
|
||||
// -----------------------------------------------------------------------
|
||||
const auto injector = di::make_injector(
|
||||
di::bind<ApplicationOptions>().to(options),
|
||||
di::bind<std::string>().to(model_path),
|
||||
di::bind<WebClient>().to<HttpWebClient>(),
|
||||
di::bind<IExportService>().to<SqliteExportService>(),
|
||||
di::bind<IPromptFormatter>().to<Gemma4JinjaPromptFormatter>(),
|
||||
di::bind<ILogger>().to(
|
||||
[log_channel](const auto&) -> std::unique_ptr<ILogger> {
|
||||
return std::make_unique<ChannelLogger>(*log_channel);
|
||||
}),
|
||||
di::bind<IEnrichmentService>().to(
|
||||
[options](const auto& inj) -> std::unique_ptr<IEnrichmentService> {
|
||||
if (options.generator.use_mocked) {
|
||||
return std::make_unique<MockEnrichmentService>();
|
||||
}
|
||||
|
||||
return std::make_unique<WikipediaEnrichmentService>(
|
||||
inj.template create<std::unique_ptr<WebClient>>());
|
||||
}),
|
||||
di::bind<DataGenerator>().to(
|
||||
[options, model_path, sampling, &prompt_directory](
|
||||
[&options, &model_path, &sampling, &prompt_directory,
|
||||
&channel_logger](
|
||||
const auto& inj) -> std::unique_ptr<DataGenerator> {
|
||||
if (options.generator.use_mocked) {
|
||||
spdlog::info(
|
||||
"[Generator] Using MockGenerator (no model path provided)");
|
||||
channel_logger.Log(
|
||||
LogLevel::Info, PipelinePhase::Startup,
|
||||
"Using MockGenerator (no model path provided)");
|
||||
return std::make_unique<MockGenerator>();
|
||||
}
|
||||
|
||||
spdlog::info(
|
||||
"[Generator] Using LlamaGenerator: {} (temperature={}, "
|
||||
"top-p={}, top-k={}, n_ctx={}, seed={})",
|
||||
model_path, sampling.temperature, sampling.top_p,
|
||||
sampling.top_k, sampling.n_ctx, sampling.seed);
|
||||
channel_logger.Log(
|
||||
LogLevel::Info, PipelinePhase::Startup,
|
||||
"Using LlamaGenerator: " + model_path +
|
||||
" (temperature=" + std::to_string(sampling.temperature) +
|
||||
", top-p=" + std::to_string(sampling.top_p) +
|
||||
", top-k=" + std::to_string(sampling.top_k) +
|
||||
", n_ctx=" + std::to_string(sampling.n_ctx) +
|
||||
", seed=" + std::to_string(sampling.seed) + ")");
|
||||
return std::make_unique<LlamaGenerator>(
|
||||
options, model_path,
|
||||
inj.template create<std::unique_ptr<IPromptFormatter>>(),
|
||||
std::move(prompt_directory));
|
||||
})
|
||||
}));
|
||||
|
||||
);
|
||||
// -----------------------------------------------------------------------
|
||||
// Pipeline execution
|
||||
// -----------------------------------------------------------------------
|
||||
const auto orchestrator =
|
||||
injector.create<std::unique_ptr<BiergartenPipelineOrchestrator>>();
|
||||
|
||||
const auto generator =
|
||||
injector.create<std::unique_ptr<BiergartenDataGenerator>>();
|
||||
|
||||
if (!generator->Run()) {
|
||||
spdlog::error("Pipeline execution failed");
|
||||
if (!orchestrator->Run()) {
|
||||
channel_logger.Log(LogLevel::Error, PipelinePhase::Teardown,
|
||||
"Pipeline execution failed");
|
||||
log_channel->Close();
|
||||
log_thread.join();
|
||||
return 1;
|
||||
}
|
||||
|
||||
spdlog::info("Pipeline executed successfully in {} ms", timer.Elapsed());
|
||||
channel_logger.Log(LogLevel::Info, PipelinePhase::Teardown,
|
||||
"Pipeline executed successfully in " +
|
||||
std::to_string(timer.Elapsed()) + " ms");
|
||||
|
||||
log_channel->Close();
|
||||
log_thread.join();
|
||||
return 0;
|
||||
|
||||
} catch (const std::exception& exception) {
|
||||
// Channel may be in an unknown state; fall back to spdlog directly.
|
||||
spdlog::critical("Unhandled fatal error in main: {}", exception.what());
|
||||
log_channel->Close();
|
||||
log_thread.join();
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
23
tooling/pipeline/src/services/logging/channel_logger.cc
Normal file
23
tooling/pipeline/src/services/logging/channel_logger.cc
Normal file
@@ -0,0 +1,23 @@
|
||||
/**
|
||||
* @file services/logging/channel_logger.cc
|
||||
*/
|
||||
|
||||
#include <chrono>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
#include "concurrency/bounded_channel.h"
|
||||
#include "services/logging/channel_logger.h"
|
||||
#include "services/logging/log_entry.h"
|
||||
|
||||
ChannelLogger::ChannelLogger(BoundedChannel<LogEntry>& channel)
|
||||
: channel_(channel) {}
|
||||
|
||||
void ChannelLogger::Log(LogLevel level, PipelinePhase phase,
|
||||
const std::string_view message) {
|
||||
channel_.Send(LogEntry{.timestamp = std::chrono::system_clock::now(),
|
||||
.level = level,
|
||||
.phase = phase,
|
||||
.message = std::string(message)});
|
||||
}
|
||||
74
tooling/pipeline/src/services/logging/log_consumer.cc
Normal file
74
tooling/pipeline/src/services/logging/log_consumer.cc
Normal file
@@ -0,0 +1,74 @@
|
||||
/**
|
||||
* @file services/logging/log_consumer.cc
|
||||
* @brief Dedicated log drain worker implementation.
|
||||
*
|
||||
* LogConsumer drains LogEntry items from a BoundedChannel and forwards them
|
||||
* to spdlog for final output.
|
||||
*/
|
||||
|
||||
#include "services/logging/log_consumer.h"
|
||||
|
||||
#include <spdlog/spdlog.h>
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "concurrency/bounded_channel.h"
|
||||
#include "services/logging/log_entry.h"
|
||||
|
||||
LogConsumer::LogConsumer(BoundedChannel<LogEntry>& channel)
|
||||
: channel_(channel) {}
|
||||
|
||||
void LogConsumer::Run() {
|
||||
while (true) {
|
||||
auto entry = channel_.Receive();
|
||||
if (!entry.has_value()) {
|
||||
// Channel is closed and drained.
|
||||
break;
|
||||
}
|
||||
|
||||
const LogEntry& log_entry = entry.value();
|
||||
auto logger = spdlog::default_logger();
|
||||
|
||||
const std::string formatted_message = [&] {
|
||||
std::string msg = std::string(log_entry.message);
|
||||
msg += " [phase=" + ToString(log_entry.phase) + "]";
|
||||
return msg;
|
||||
}();
|
||||
|
||||
logger->log(ToSpdlogLevel(log_entry.level), formatted_message);
|
||||
}
|
||||
}
|
||||
|
||||
spdlog::level::level_enum LogConsumer::ToSpdlogLevel(LogLevel level) {
|
||||
switch (level) {
|
||||
case LogLevel::Debug:
|
||||
return spdlog::level::debug;
|
||||
case LogLevel::Info:
|
||||
return spdlog::level::info;
|
||||
case LogLevel::Warn:
|
||||
return spdlog::level::warn;
|
||||
case LogLevel::Error:
|
||||
return spdlog::level::err;
|
||||
}
|
||||
return spdlog::level::info;
|
||||
}
|
||||
|
||||
std::string LogConsumer::ToString(PipelinePhase phase) {
|
||||
switch (phase) {
|
||||
case PipelinePhase::Startup:
|
||||
return "Startup";
|
||||
case PipelinePhase::UserGeneration:
|
||||
return "UserGeneration";
|
||||
case PipelinePhase::BreweryAndBeerGeneration:
|
||||
return "BreweryAndBeerGeneration";
|
||||
case PipelinePhase::CheckinGeneration:
|
||||
return "CheckinGeneration";
|
||||
case PipelinePhase::RatingGeneration:
|
||||
return "RatingGeneration";
|
||||
case PipelinePhase::FollowGeneration:
|
||||
return "FollowGeneration";
|
||||
case PipelinePhase::Teardown:
|
||||
return "Teardown";
|
||||
}
|
||||
return "Unknown";
|
||||
}
|
||||
Reference in New Issue
Block a user