mirror of
https://github.com/aaronpo97/the-biergarten-app.git
synced 2026-07-16 09:37:23 +00:00
style: update include guards and incl statements to follow google style guide
This commit is contained in:
70
tooling/pipeline/includes/biergarten_pipeline.h
Normal file
70
tooling/pipeline/includes/biergarten_pipeline.h
Normal file
@@ -0,0 +1,70 @@
|
|||||||
|
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_
|
||||||
|
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @file biergarten_pipeline.h
|
||||||
|
* @brief Master umbrella header — includes every public header in the
|
||||||
|
* Biergarten pipeline.
|
||||||
|
*/
|
||||||
|
|
||||||
|
// --- orchestrator ---
|
||||||
|
#include "biergarten_pipeline_orchestrator.h"
|
||||||
|
|
||||||
|
// --- concurrency ---
|
||||||
|
#include "concurrency/bounded_channel.h"
|
||||||
|
|
||||||
|
// --- data_generation ---
|
||||||
|
#include "data_generation/data_generator.h"
|
||||||
|
#include "data_generation/json_grammars.h"
|
||||||
|
#include "data_generation/llama_generator.h"
|
||||||
|
#include "data_generation/llama_generator_helpers.h"
|
||||||
|
#include "data_generation/mock_generator.h"
|
||||||
|
#include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h"
|
||||||
|
#include "data_generation/prompt_formatting/prompt_formatter.h"
|
||||||
|
|
||||||
|
// --- data_model ---
|
||||||
|
#include "data_model/generated_models.h"
|
||||||
|
#include "data_model/models.h"
|
||||||
|
#include "data_model/names_by_country.h"
|
||||||
|
|
||||||
|
// --- json_handling ---
|
||||||
|
#include "json_handling/json_loader.h"
|
||||||
|
#include "json_handling/pretty_print.h"
|
||||||
|
|
||||||
|
// --- llama backend ---
|
||||||
|
#include "llama_backend_state.h"
|
||||||
|
|
||||||
|
// --- services: curated_data ---
|
||||||
|
#include "services/curated_data/curated_data_service.h"
|
||||||
|
|
||||||
|
// --- services: database ---
|
||||||
|
#include "services/database/export_service.h"
|
||||||
|
#include "services/database/sqlite_connection_helpers.h"
|
||||||
|
#include "services/database/sqlite_export_service.h"
|
||||||
|
#include "services/database/sqlite_export_service_helpers.h"
|
||||||
|
#include "services/database/sqlite_handle_types.h"
|
||||||
|
#include "services/database/sqlite_statement_helpers.h"
|
||||||
|
|
||||||
|
// --- services: datetime ---
|
||||||
|
#include "services/datetime/date_time_provider.h"
|
||||||
|
#include "services/datetime/timer.h"
|
||||||
|
|
||||||
|
// --- services: enrichment ---
|
||||||
|
#include "services/enrichment/enrichment_service.h"
|
||||||
|
#include "services/enrichment/mock_enrichment.h"
|
||||||
|
#include "services/enrichment/wikipedia_service.h"
|
||||||
|
|
||||||
|
// --- services: logging ---
|
||||||
|
#include "services/logging/log_dispatcher.h"
|
||||||
|
#include "services/logging/log_entry.h"
|
||||||
|
#include "services/logging/log_producer.h"
|
||||||
|
#include "services/logging/logger.h"
|
||||||
|
|
||||||
|
// --- services: prompting ---
|
||||||
|
#include "services/prompting/prompt_directory.h"
|
||||||
|
|
||||||
|
// --- web_client ---
|
||||||
|
#include "web_client/http_web_client.h"
|
||||||
|
#include "web_client/web_client.h"
|
||||||
|
|
||||||
|
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_
|
||||||
@@ -1,12 +1,9 @@
|
|||||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_
|
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_
|
||||||
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_
|
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @file biergarten_pipeline_orchestrator.h
|
* @file biergarten_pipeline_orchestrator.h
|
||||||
* @brief Orchestration for end-to-end brewery and user data generation.
|
* @brief Orchestration for end-to-end brewery and user data generation.
|
||||||
*
|
|
||||||
* Coordinates location loading, enrichment, and generation phases to produce
|
|
||||||
* a complete dataset, wiring dependencies selected at the composition root.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include <memory>
|
#include <memory>
|
||||||
@@ -103,10 +100,6 @@ class BiergartenPipelineOrchestrator {
|
|||||||
* @brief Generate users grounded in sampled names and personas for
|
* @brief Generate users grounded in sampled names and personas for
|
||||||
* enriched cities.
|
* enriched cities.
|
||||||
*
|
*
|
||||||
* Loads personas.json / forenames-by-country.json /
|
|
||||||
* surnames-by-country.json itself, mirroring how QueryCitiesWithCountries()
|
|
||||||
* owns its own locations.json load.
|
|
||||||
*
|
|
||||||
* @param cities Span of enriched city data.
|
* @param cities Span of enriched city data.
|
||||||
*/
|
*/
|
||||||
void GenerateUsers(std::span<const EnrichedCity> cities);
|
void GenerateUsers(std::span<const EnrichedCity> cities);
|
||||||
@@ -120,4 +113,4 @@ class BiergartenPipelineOrchestrator {
|
|||||||
std::vector<UserRecord> generated_users_;
|
std::vector<UserRecord> generated_users_;
|
||||||
};
|
};
|
||||||
|
|
||||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_
|
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
#include "bounded_channel.h"
|
#include "concurrency/bounded_channel.h"
|
||||||
|
|
||||||
template <typename T>
|
template <typename T>
|
||||||
void BoundedChannel<T>::Send(T item) {
|
void BoundedChannel<T>::Send(T item) {
|
||||||
|
|||||||
@@ -1,24 +1,23 @@
|
|||||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_
|
#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_
|
||||||
#define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_
|
#define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_
|
||||||
|
|
||||||
#include <filesystem>
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @file data_generation/llama_generator.h
|
* @file data_generation/llama_generator.h
|
||||||
* @brief llama.cpp-backed implementation of DataGenerator.
|
* @brief llama.cpp-backed implementation of DataGenerator.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include <cstdint>
|
#include <cstdint>
|
||||||
|
#include <filesystem>
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include <random>
|
#include <random>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <string_view>
|
#include <string_view>
|
||||||
|
|
||||||
#include "../services/prompting/prompt_directory.h"
|
|
||||||
#include "data_generation/data_generator.h"
|
#include "data_generation/data_generator.h"
|
||||||
#include "data_generation/prompt_formatting/prompt_formatter.h"
|
#include "data_generation/prompt_formatting/prompt_formatter.h"
|
||||||
#include "data_model/models.h"
|
#include "data_model/models.h"
|
||||||
#include "services/logging/logger.h"
|
#include "services/logging/logger.h"
|
||||||
|
#include "services/prompting/prompt_directory.h"
|
||||||
|
|
||||||
struct llama_model;
|
struct llama_model;
|
||||||
struct llama_context;
|
struct llama_context;
|
||||||
|
|||||||
@@ -7,7 +7,6 @@
|
|||||||
* inputs.
|
* inputs.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include <boost/program_options.hpp>
|
|
||||||
#include <cstdint>
|
#include <cstdint>
|
||||||
#include <filesystem>
|
#include <filesystem>
|
||||||
#include <memory>
|
#include <memory>
|
||||||
@@ -16,6 +15,8 @@
|
|||||||
#include <string_view>
|
#include <string_view>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
|
#include <boost/program_options.hpp>
|
||||||
|
|
||||||
class ILogger;
|
class ILogger;
|
||||||
|
|
||||||
namespace prog_opts = boost::program_options;
|
namespace prog_opts = boost::program_options;
|
||||||
|
|||||||
@@ -9,10 +9,11 @@
|
|||||||
* readable output. Adapted from Boost JSON library examples.
|
* readable output. Adapted from Boost JSON library examples.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include <boost/json.hpp>
|
|
||||||
#include <ostream>
|
#include <ostream>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
|
#include <boost/json.hpp>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Pretty-prints a JSON value to an output stream with indentation.
|
* @brief Pretty-prints a JSON value to an output stream with indentation.
|
||||||
*
|
*
|
||||||
@@ -106,4 +107,4 @@ inline void PrettyPrint(std::ostream& outstream,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#endif
|
#endif // BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_PRETTY_PRINT_H_
|
||||||
|
|||||||
@@ -6,13 +6,13 @@
|
|||||||
* @brief Declarations for connection-level SQLite helper functions.
|
* @brief Declarations for connection-level SQLite helper functions.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include <sqlite3.h>
|
|
||||||
|
|
||||||
#include <filesystem>
|
#include <filesystem>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <string_view>
|
#include <string_view>
|
||||||
|
|
||||||
#include "sqlite_handle_types.h"
|
#include <sqlite3.h>
|
||||||
|
|
||||||
|
#include "services/database/sqlite_handle_types.h"
|
||||||
|
|
||||||
namespace sqlite_export_service_internal {
|
namespace sqlite_export_service_internal {
|
||||||
|
|
||||||
|
|||||||
@@ -11,10 +11,10 @@
|
|||||||
#include <string>
|
#include <string>
|
||||||
#include <unordered_map>
|
#include <unordered_map>
|
||||||
|
|
||||||
#include "../datetime/date_time_provider.h"
|
|
||||||
#include "data_model/models.h"
|
#include "data_model/models.h"
|
||||||
#include "export_service.h"
|
#include "services/database/export_service.h"
|
||||||
#include "sqlite_export_service_helpers.h"
|
#include "services/database/sqlite_export_service_helpers.h"
|
||||||
|
#include "services/datetime/date_time_provider.h"
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Persists generated brewery records into a fresh SQLite database.
|
* @brief Persists generated brewery records into a fresh SQLite database.
|
||||||
|
|||||||
@@ -3,8 +3,8 @@
|
|||||||
|
|
||||||
/* Umbrella header for backward compatibility. */
|
/* Umbrella header for backward compatibility. */
|
||||||
|
|
||||||
#include "sqlite_connection_helpers.h"
|
#include "services/database/sqlite_connection_helpers.h"
|
||||||
#include "sqlite_handle_types.h"
|
#include "services/database/sqlite_handle_types.h"
|
||||||
#include "sqlite_statement_helpers.h"
|
#include "services/database/sqlite_statement_helpers.h"
|
||||||
|
|
||||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATABASE_SQLITE_EXPORT_SERVICE_HELPERS_H_
|
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATABASE_SQLITE_EXPORT_SERVICE_HELPERS_H_
|
||||||
|
|||||||
@@ -5,11 +5,11 @@
|
|||||||
* Shared handle and parameter type declarations used by SQLite helper units.
|
* Shared handle and parameter type declarations used by SQLite helper units.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include <sqlite3.h>
|
|
||||||
|
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include <string_view>
|
#include <string_view>
|
||||||
|
|
||||||
|
#include <sqlite3.h>
|
||||||
|
|
||||||
namespace sqlite_export_service_internal {
|
namespace sqlite_export_service_internal {
|
||||||
|
|
||||||
struct SqliteDatabaseDeleter {
|
struct SqliteDatabaseDeleter {
|
||||||
|
|||||||
@@ -7,13 +7,13 @@
|
|||||||
* constants.
|
* constants.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include <sqlite3.h>
|
|
||||||
|
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <string_view>
|
#include <string_view>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "sqlite_handle_types.h"
|
#include <sqlite3.h>
|
||||||
|
|
||||||
|
#include "services/database/sqlite_handle_types.h"
|
||||||
|
|
||||||
namespace sqlite_export_service_internal {
|
namespace sqlite_export_service_internal {
|
||||||
|
|
||||||
@@ -109,30 +109,36 @@ INSERT INTO users (
|
|||||||
|
|
||||||
// sqlite3_bind_*() parameter indices are 1-based, matching the "?"
|
// sqlite3_bind_*() parameter indices are 1-based, matching the "?"
|
||||||
// placeholder order in the SQL above.
|
// placeholder order in the SQL above.
|
||||||
inline constexpr int kLocationCityBindIndex = 1;
|
enum eLocationBindIndex {
|
||||||
inline constexpr int kLocationStateProvinceBindIndex = 2;
|
kLocationCityBindIndex = 1,
|
||||||
inline constexpr int kLocationIso31662BindIndex = 3;
|
kLocationStateProvinceBindIndex,
|
||||||
inline constexpr int kLocationCountryBindIndex = 4;
|
kLocationIso31662BindIndex,
|
||||||
inline constexpr int kLocationIso31661BindIndex = 5;
|
kLocationCountryBindIndex,
|
||||||
inline constexpr int kLocationLanguagesBindIndex = 6;
|
kLocationIso31661BindIndex,
|
||||||
inline constexpr int kLocationLatitudeBindIndex = 7;
|
kLocationLanguagesBindIndex,
|
||||||
inline constexpr int kLocationLongitudeBindIndex = 8;
|
kLocationLatitudeBindIndex,
|
||||||
|
kLocationLongitudeBindIndex,
|
||||||
|
};
|
||||||
|
|
||||||
inline constexpr int kBreweryLocationIdBindIndex = 1;
|
enum BreweryBindIndex {
|
||||||
inline constexpr int kBreweryEnglishNameBindIndex = 2;
|
kBreweryLocationIdBindIndex = 1,
|
||||||
inline constexpr int kBreweryEnglishDescriptionBindIndex = 3;
|
kBreweryEnglishNameBindIndex,
|
||||||
inline constexpr int kBreweryLocalNameBindIndex = 4;
|
kBreweryEnglishDescriptionBindIndex,
|
||||||
inline constexpr int kBreweryLocalDescriptionBindIndex = 5;
|
kBreweryLocalNameBindIndex,
|
||||||
|
kBreweryLocalDescriptionBindIndex,
|
||||||
|
};
|
||||||
|
|
||||||
inline constexpr int kUserLocationIdBindIndex = 1;
|
enum UserBindIndex {
|
||||||
inline constexpr int kUserFirstNameBindIndex = 2;
|
kUserLocationIdBindIndex = 1,
|
||||||
inline constexpr int kUserLastNameBindIndex = 3;
|
kUserFirstNameBindIndex,
|
||||||
inline constexpr int kUserGenderBindIndex = 4;
|
kUserLastNameBindIndex,
|
||||||
inline constexpr int kUserUsernameBindIndex = 5;
|
kUserGenderBindIndex,
|
||||||
inline constexpr int kUserBioBindIndex = 6;
|
kUserUsernameBindIndex,
|
||||||
inline constexpr int kUserActivityWeightBindIndex = 7;
|
kUserBioBindIndex,
|
||||||
inline constexpr int kUserEmailBindIndex = 8;
|
kUserActivityWeightBindIndex,
|
||||||
inline constexpr int kUserDateOfBirthBindIndex = 9;
|
kUserEmailBindIndex,
|
||||||
|
kUserDateOfBirthBindIndex,
|
||||||
|
};
|
||||||
|
|
||||||
SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle,
|
SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle,
|
||||||
std::string_view sql,
|
std::string_view sql,
|
||||||
|
|||||||
@@ -8,7 +8,7 @@
|
|||||||
|
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
#include "enrichment_service.h"
|
#include "services/enrichment/enrichment_service.h"
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Enrichment service that returns no context for any location.
|
* @brief Enrichment service that returns no context for any location.
|
||||||
|
|||||||
@@ -11,7 +11,7 @@
|
|||||||
#include <string_view>
|
#include <string_view>
|
||||||
#include <unordered_map>
|
#include <unordered_map>
|
||||||
|
|
||||||
#include "enrichment_service.h"
|
#include "services/enrichment/enrichment_service.h"
|
||||||
#include "services/logging/logger.h"
|
#include "services/logging/logger.h"
|
||||||
#include "web_client/web_client.h"
|
#include "web_client/web_client.h"
|
||||||
|
|
||||||
|
|||||||
@@ -6,8 +6,8 @@
|
|||||||
* a bounded channel for later processing by the dispatcher.
|
* a bounded channel for later processing by the dispatcher.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_
|
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_
|
||||||
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_
|
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_
|
||||||
|
|
||||||
#include <string_view>
|
#include <string_view>
|
||||||
|
|
||||||
@@ -50,4 +50,4 @@ class LogProducer final : public ILogger {
|
|||||||
BoundedChannel<LogEntry>& channel_;
|
BoundedChannel<LogEntry>& channel_;
|
||||||
};
|
};
|
||||||
|
|
||||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_
|
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_
|
||||||
|
|||||||
@@ -52,4 +52,4 @@ class HttpWebClient final : public WebClient {
|
|||||||
std::shared_ptr<ILogger> logger_;
|
std::shared_ptr<ILogger> logger_;
|
||||||
};
|
};
|
||||||
|
|
||||||
#endif
|
#endif // BIERGARTEN_PIPELINE_INCLUDES_WEB_CLIENT_HTTP_WEB_CLIENT_H_
|
||||||
|
|||||||
@@ -19,5 +19,4 @@ BiergartenPipelineOrchestrator::BiergartenPipelineOrchestrator(
|
|||||||
generator_(std::move(generator)),
|
generator_(std::move(generator)),
|
||||||
exporter_(std::move(exporter)),
|
exporter_(std::move(exporter)),
|
||||||
curated_data_service_(std::move(curated_data_service)),
|
curated_data_service_(std::move(curated_data_service)),
|
||||||
application_options_(application_options) {
|
application_options_(application_options) {}
|
||||||
}
|
|
||||||
@@ -86,8 +86,8 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
|
|||||||
}
|
}
|
||||||
|
|
||||||
const NamesByCountry names_by_country =
|
const NamesByCountry names_by_country =
|
||||||
curated_data_service_->LoadNamesByCountry(
|
curated_data_service_->LoadNamesByCountry("forenames-by-country.json",
|
||||||
"forenames-by-country.json", "surnames-by-country.json");
|
"surnames-by-country.json");
|
||||||
|
|
||||||
std::mt19937 rng(std::random_device{}());
|
std::mt19937 rng(std::random_device{}());
|
||||||
std::uniform_int_distribution<size_t> persona_dist(0, personas.size() - 1);
|
std::uniform_int_distribution<size_t> persona_dist(0, personas.size() - 1);
|
||||||
|
|||||||
@@ -3,12 +3,13 @@
|
|||||||
* @brief BiergartenPipelineOrchestrator::LogResults() implementation.
|
* @brief BiergartenPipelineOrchestrator::LogResults() implementation.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include <boost/json/array.hpp>
|
|
||||||
#include <chrono>
|
#include <chrono>
|
||||||
#include <format>
|
#include <format>
|
||||||
|
|
||||||
#include "../../includes/json_handling/pretty_print.h"
|
#include <boost/json/array.hpp>
|
||||||
|
|
||||||
#include "biergarten_pipeline_orchestrator.h"
|
#include "biergarten_pipeline_orchestrator.h"
|
||||||
|
#include "json_handling/pretty_print.h"
|
||||||
#include "services/logging/logger.h"
|
#include "services/logging/logger.h"
|
||||||
|
|
||||||
void BiergartenPipelineOrchestrator::LogResults() const {
|
void BiergartenPipelineOrchestrator::LogResults() const {
|
||||||
|
|||||||
@@ -6,7 +6,6 @@
|
|||||||
|
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include <array>
|
#include <array>
|
||||||
#include <boost/json.hpp>
|
|
||||||
#include <cctype>
|
#include <cctype>
|
||||||
#include <optional>
|
#include <optional>
|
||||||
#include <span>
|
#include <span>
|
||||||
@@ -15,8 +14,11 @@
|
|||||||
#include <string_view>
|
#include <string_view>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
|
#include <boost/json.hpp>
|
||||||
|
#include <llama.h>
|
||||||
|
|
||||||
#include "data_generation/llama_generator_helpers.h"
|
#include "data_generation/llama_generator_helpers.h"
|
||||||
#include "llama.h"
|
|
||||||
namespace {
|
namespace {
|
||||||
/**
|
/**
|
||||||
* String trimming: removes leading and trailing whitespace
|
* String trimming: removes leading and trailing whitespace
|
||||||
|
|||||||
@@ -14,9 +14,10 @@
|
|||||||
#include <string_view>
|
#include <string_view>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
|
#include <llama.h>
|
||||||
|
|
||||||
#include "data_generation/llama_generator.h"
|
#include "data_generation/llama_generator.h"
|
||||||
#include "data_generation/llama_generator_helpers.h"
|
#include "data_generation/llama_generator_helpers.h"
|
||||||
#include "llama.h"
|
|
||||||
|
|
||||||
static constexpr size_t kPromptTokenSlack = 8;
|
static constexpr size_t kPromptTokenSlack = 8;
|
||||||
// Minimum tokens to keep when using top-p sampling. Ensures at least one
|
// Minimum tokens to keep when using top-p sampling. Ensures at least one
|
||||||
|
|||||||
@@ -11,8 +11,9 @@
|
|||||||
#include <stdexcept>
|
#include <stdexcept>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
|
#include <llama.h>
|
||||||
|
|
||||||
#include "data_model/models.h"
|
#include "data_model/models.h"
|
||||||
#include "llama.h"
|
|
||||||
|
|
||||||
static constexpr uint32_t kMaxContextSize = 32768U;
|
static constexpr uint32_t kMaxContextSize = 32768U;
|
||||||
|
|
||||||
|
|||||||
@@ -10,9 +10,10 @@
|
|||||||
#include <string>
|
#include <string>
|
||||||
#include <utility>
|
#include <utility>
|
||||||
|
|
||||||
|
#include <ggml-backend.h>
|
||||||
|
#include <llama.h>
|
||||||
|
|
||||||
#include "data_generation/llama_generator.h"
|
#include "data_generation/llama_generator.h"
|
||||||
#include "ggml-backend.h"
|
|
||||||
#include "llama.h"
|
|
||||||
|
|
||||||
// Maximum batch size for decode operations. Capping the batch prevents
|
// Maximum batch size for decode operations. Capping the batch prevents
|
||||||
// excessive memory allocation while maintaining inference performance.
|
// excessive memory allocation while maintaining inference performance.
|
||||||
|
|||||||
@@ -6,7 +6,6 @@
|
|||||||
|
|
||||||
#include "json_handling/json_loader.h"
|
#include "json_handling/json_loader.h"
|
||||||
|
|
||||||
#include <boost/json.hpp>
|
|
||||||
#include <format>
|
#include <format>
|
||||||
#include <fstream>
|
#include <fstream>
|
||||||
#include <iostream>
|
#include <iostream>
|
||||||
@@ -17,6 +16,8 @@
|
|||||||
#include <unordered_map>
|
#include <unordered_map>
|
||||||
#include <utility>
|
#include <utility>
|
||||||
|
|
||||||
|
#include <boost/json.hpp>
|
||||||
|
|
||||||
#include "services/logging/logger.h"
|
#include "services/logging/logger.h"
|
||||||
|
|
||||||
static std::string ReadRequiredString(const boost::json::object& object,
|
static std::string ReadRequiredString(const boost::json::object& object,
|
||||||
@@ -137,7 +138,6 @@ std::vector<Location> JsonLoader::LoadLocations(
|
|||||||
.longitude = ReadRequiredNumber(object, "longitude"),
|
.longitude = ReadRequiredNumber(object, "longitude"),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
return locations;
|
return locations;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,11 +4,6 @@
|
|||||||
* initializes shared infrastructure, and executes the pipeline entry flow.
|
* initializes shared infrastructure, and executes the pipeline entry flow.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include <spdlog/fmt/fmt.h>
|
|
||||||
#include <spdlog/spdlog.h>
|
|
||||||
|
|
||||||
#include <boost/di.hpp>
|
|
||||||
#include <boost/program_options.hpp>
|
|
||||||
#include <chrono>
|
#include <chrono>
|
||||||
#include <exception>
|
#include <exception>
|
||||||
#include <format>
|
#include <format>
|
||||||
@@ -18,27 +13,12 @@
|
|||||||
#include <string>
|
#include <string>
|
||||||
#include <thread>
|
#include <thread>
|
||||||
|
|
||||||
#include "biergarten_pipeline_orchestrator.h"
|
#include <boost/di.hpp>
|
||||||
#include "concurrency/bounded_channel.h"
|
#include <boost/program_options.hpp>
|
||||||
#include "data_generation/llama_generator.h"
|
#include <spdlog/fmt/fmt.h>
|
||||||
#include "data_generation/mock_generator.h"
|
#include <spdlog/spdlog.h>
|
||||||
#include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h"
|
|
||||||
#include "data_model/models.h"
|
#include "biergarten_pipeline.h"
|
||||||
#include "json_handling/json_loader.h"
|
|
||||||
#include "llama_backend_state.h"
|
|
||||||
#include "services/curated_data/curated_data_service.h"
|
|
||||||
#include "services/database/export_service.h"
|
|
||||||
#include "services/database/sqlite_export_service.h"
|
|
||||||
#include "services/datetime/timer.h"
|
|
||||||
#include "services/enrichment/enrichment_service.h"
|
|
||||||
#include "services/enrichment/mock_enrichment.h"
|
|
||||||
#include "services/enrichment/wikipedia_service.h"
|
|
||||||
#include "services/logging/log_dispatcher.h"
|
|
||||||
#include "services/logging/log_entry.h"
|
|
||||||
#include "services/logging/log_producer.h"
|
|
||||||
#include "services/logging/logger.h"
|
|
||||||
#include "services/prompting/prompt_directory.h"
|
|
||||||
#include "web_client/http_web_client.h"
|
|
||||||
|
|
||||||
namespace di = boost::di;
|
namespace di = boost::di;
|
||||||
|
|
||||||
@@ -164,8 +144,7 @@ int main(const int argc, char** argv) {
|
|||||||
}),
|
}),
|
||||||
di::bind<DataGenerator>().to(
|
di::bind<DataGenerator>().to(
|
||||||
[&options, &model_path, &sampling, &prompt_directory,
|
[&options, &model_path, &sampling, &prompt_directory,
|
||||||
&log_producer
|
&log_producer](const auto& inj) -> std::unique_ptr<DataGenerator> {
|
||||||
](const auto& inj) -> std::unique_ptr<DataGenerator> {
|
|
||||||
if (options.generator.use_mocked) {
|
if (options.generator.use_mocked) {
|
||||||
log_producer->Log({.level = LogLevel::Info,
|
log_producer->Log({.level = LogLevel::Info,
|
||||||
.phase = PipelinePhase::Startup,
|
.phase = PipelinePhase::Startup,
|
||||||
|
|||||||
@@ -3,13 +3,14 @@
|
|||||||
* @brief WikipediaEnrichmentService::FetchExtract() implementation.
|
* @brief WikipediaEnrichmentService::FetchExtract() implementation.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include <boost/json.hpp>
|
|
||||||
#include <chrono>
|
#include <chrono>
|
||||||
#include <format>
|
#include <format>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <string_view>
|
#include <string_view>
|
||||||
#include <thread>
|
#include <thread>
|
||||||
|
|
||||||
|
#include <boost/json.hpp>
|
||||||
|
|
||||||
#include "services/enrichment/wikipedia_service.h"
|
#include "services/enrichment/wikipedia_service.h"
|
||||||
|
|
||||||
using namespace boost;
|
using namespace boost;
|
||||||
|
|||||||
@@ -6,10 +6,10 @@
|
|||||||
*/
|
*/
|
||||||
#include "services/logging/log_dispatcher.h"
|
#include "services/logging/log_dispatcher.h"
|
||||||
|
|
||||||
#include <spdlog/spdlog.h>
|
|
||||||
|
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
|
#include <spdlog/spdlog.h>
|
||||||
|
|
||||||
#include "concurrency/bounded_channel.h"
|
#include "concurrency/bounded_channel.h"
|
||||||
#include "services/logging/log_entry.h"
|
#include "services/logging/log_entry.h"
|
||||||
|
|
||||||
|
|||||||
@@ -1,11 +1,12 @@
|
|||||||
#include "services/database/sqlite_statement_helpers.h"
|
#include "services/database/sqlite_statement_helpers.h"
|
||||||
|
|
||||||
#include <boost/json.hpp>
|
|
||||||
#include <cstring>
|
#include <cstring>
|
||||||
#include <limits>
|
#include <limits>
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include <stdexcept>
|
#include <stdexcept>
|
||||||
|
|
||||||
|
#include <boost/json.hpp>
|
||||||
|
|
||||||
#include "services/database/sqlite_connection_helpers.h"
|
#include "services/database/sqlite_connection_helpers.h"
|
||||||
|
|
||||||
namespace sqlite_export_service_internal {
|
namespace sqlite_export_service_internal {
|
||||||
|
|||||||
@@ -5,8 +5,6 @@
|
|||||||
|
|
||||||
#include "web_client/http_web_client.h"
|
#include "web_client/http_web_client.h"
|
||||||
|
|
||||||
#include <httplib.h>
|
|
||||||
|
|
||||||
#include <chrono>
|
#include <chrono>
|
||||||
#include <format>
|
#include <format>
|
||||||
#include <regex>
|
#include <regex>
|
||||||
@@ -14,6 +12,8 @@
|
|||||||
#include <string>
|
#include <string>
|
||||||
#include <utility>
|
#include <utility>
|
||||||
|
|
||||||
|
#include <httplib.h>
|
||||||
|
|
||||||
#include "services/logging/logger.h"
|
#include "services/logging/logger.h"
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
|
|||||||
Reference in New Issue
Block a user