AaronPo/the-biergarten-app
1
0
Fork 0
mirror of https://github.com/aaronpo97/the-biergarten-app.git synced 2026-10-14 17:54:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Pipeline: City rename, add address/postal-code generation, add xeger generation (#232)

Browse Source
This commit is contained in:
Aaron Po
2026-07-19 03:59:06 -04:00
parent 901f30ffd8
commit 8e7ecf2fc3
56 changed files with 2454 additions and 1151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
docs/pipeline/ETHICS-AND-KNOWN-ISSUES.md
View File

@@ -18,6 +18,7 @@ low-resource language failures.
- [Known Issues](#known-issues)
- [Hallucinated Brewing Techniques](#hallucinated-brewing-techniques)
- [Low-Resource Language Hallucination](#low-resource-language-hallucination)
- [Synthetic Postal Codes](#synthetic-postal-codes)
---
@@ -286,6 +287,21 @@ Output sample:
SQLite schema so downstream applications can filter or flag potentially
hallucinated text by language tier.
### Synthetic Postal Codes
When the pipeline is run without `--mocked`, `XegerPostalCodeService` generates
a fresh postal code per brewery/user address by reverse-matching one of the
city's curated postal-code regexes from `locations.json` (falling back to the
country format regex), rather than returning one of `MockPostalCodeService`'s
curated real-world examples.
The generated code is **format-conformant by construction** — it always
satisfies the regex it was generated from — but it is not looked up from any
real postal registry. It does not correspond to a real address, business, or
resident, and should not be treated as accurate geocoding or delivery data.
This is consistent with the rest of the dataset: fixture data for a
proof-of-concept, not a source of truth.
---
## Footnotes

15
tooling/pipeline/CMakeLists.txt
View File

@@ -111,6 +111,8 @@ if(NOT BIERGARTEN_MOCK_ONLY)
)
FetchContent_MakeAvailable(llama-cpp)
endif()
find_package(OpenMP REQUIRED)
endif()
# spdlog
@@ -208,6 +210,18 @@ target_sources(${PROJECT_NAME} PRIVATE
src/services/enrichment/wikipedia/get_summary.cc
)
# --- services: postal_code ---
target_sources(${PROJECT_NAME} PRIVATE
src/services/postal_code/xeger_postal_code_service.cc
)
# --- services: xeger ---
target_sources(${PROJECT_NAME} PRIVATE
src/services/xeger/xeger.cc
src/services/xeger/xeger_parser.cc
src/services/xeger/xeger_generator.cc
)
# --- services: sqlite ---
target_sources(${PROJECT_NAME} PRIVATE
src/services/sqlite/process_record.cc
@@ -237,6 +251,7 @@ target_include_directories(${PROJECT_NAME} PRIVATE
target_link_libraries(${PROJECT_NAME} PRIVATE
$<$<NOT:$<BOOL:${BIERGARTEN_MOCK_ONLY}>>:llama>
$<$<NOT:$<BOOL:${BIERGARTEN_MOCK_ONLY}>>:OpenMP::OpenMP_CXX>
boost::di
Boost::json
Boost::program_options

6
tooling/pipeline/includes/biergarten_pipeline.h
View File

@@ -58,6 +58,12 @@
#include "services/enrichment/mock_enrichment.h"
#include "services/enrichment/wikipedia_service.h"
// --- services: postal_code ---
#include "services/postal_code/mock_postal_code_service.h"
#include "services/postal_code/postal_code_service.h"
#include "services/postal_code/xeger_postal_code_service.h"
#include "services/xeger/xeger.h"
// --- services: logging ---
#include "services/logging/log_dispatcher.h"
#include "services/logging/log_entry.h"

7
tooling/pipeline/includes/biergarten_pipeline_orchestrator.h
View File

@@ -16,6 +16,7 @@
#include "services/database/export_service.h"
#include "services/enrichment/enrichment_service.h"
#include "services/logging/logger.h"
#include "services/postal_code/postal_code_service.h"
/**
* @brief Main orchestrator for the Biergarten data generation pipeline.
@@ -34,6 +35,8 @@ class BiergartenPipelineOrchestrator {
* @param exporter Database backend for persisting generated records.
* @param curated_data_service Loads curated location, persona, and name
* data used to seed generation.
* @param postal_code_service Resolves a postal code for a city from its
* curated postal-code data.
* @param application_options CLI configuration and paths.
*/
BiergartenPipelineOrchestrator(
@@ -42,6 +45,7 @@ class BiergartenPipelineOrchestrator {
std::unique_ptr<DataGenerator> generator,
std::unique_ptr<IExportService> exporter,
std::unique_ptr<ICuratedDataService> curated_data_service,
std::unique_ptr<IPostalCodeService> postal_code_service,
const ApplicationOptions& application_options);
/**
@@ -78,6 +82,7 @@ class BiergartenPipelineOrchestrator {
*/
std::unique_ptr<IExportService> exporter_;
std::unique_ptr<ICuratedDataService> curated_data_service_;
std::unique_ptr<IPostalCodeService> postal_code_service_;
ApplicationOptions application_options_;
@@ -87,7 +92,7 @@ class BiergartenPipelineOrchestrator {
* @return Vector of locations randomly sampled per
* PipelineOptions::location_count.
*/
std::vector<Location> QueryCitiesWithCountries();
std::vector<City> QueryLocations();
/**
* @brief Generate breweries for enriched cities.

2
tooling/pipeline/includes/concurrency/bounded_channel.h
View File

@@ -26,9 +26,7 @@
*/
template <typename T>
class BoundedChannel {
// -------------------------------------------------------------------------
// Internal state — all access must be guarded by mutex_.
// -------------------------------------------------------------------------
std::queue<T> queue_;

6
tooling/pipeline/includes/data_generation/data_generator.h
View File

@@ -20,12 +20,10 @@ class DataGenerator {
/**
* @brief Generates brewery data for a location.
*
* @param location Location data
* @param region_context Additional regional context text.
* @param city Enriched city the brewery is associated with.
* @return Brewery generation result.
*/
virtual BreweryResult GenerateBrewery(const Location& location,
const std::string& region_context) = 0;
virtual BreweryResult GenerateBrewery(const EnrichedCity& city) = 0;
/**
* @brief Generates a user profile grounded in a sampled name and persona.

5
tooling/pipeline/includes/data_generation/llama_generator.h
View File

@@ -51,12 +51,11 @@ class LlamaGenerator final : public DataGenerator {
/**
* @brief Generates brewery data for a specific location.
*
* @param location Location object.
* @param location City object.
* @param region_context Additional regional context.
* @return Generated brewery result.
*/
BreweryResult GenerateBrewery(const Location& location,
const std::string& region_context) override;
BreweryResult GenerateBrewery(const EnrichedCity&) override;
/**
* @brief Generates a user profile grounded in a sampled name and persona.

15
tooling/pipeline/includes/data_generation/mock_generator.h
View File

@@ -20,20 +20,15 @@ class MockGenerator final : public DataGenerator {
/**
* @brief Generates deterministic brewery data for a location.
*
* @param location City and country names.
* @param region_context Unused for mock generation.
* @param enriched_city
* @return Generated brewery result.
*/
BreweryResult GenerateBrewery(const Location& location,
const std::string& region_context) override;
BreweryResult GenerateBrewery(const EnrichedCity& enriched_city) override;
/**
* @brief Generates deterministic user data grounded in a sampled name and
* persona.
*
* @param city Enriched city the user is associated with.
* @param persona Persona archetype.
* @param name Sampled first/last name, copied directly into the result.
* @return Generated user result.
*/
UserResult GenerateUser(const EnrichedCity& city, const UserPersona& persona,
@@ -46,7 +41,7 @@ class MockGenerator final : public DataGenerator {
* @param location City and country names.
* @return Deterministic hash value.
*/
static size_t DeterministicHash(const Location& location);
static size_t DeterministicHash(const City& location);
/**
* @brief Combines city, persona, and name into a stable hash value.
@@ -56,7 +51,7 @@ class MockGenerator final : public DataGenerator {
* @param name Sampled first/last name.
* @return Deterministic hash value.
*/
static size_t DeterministicHash(const Location& location,
static size_t DeterministicHash(const City& location,
const UserPersona& persona, const Name& name);
// Hash stride constants for deterministic distribution across fixed-size

28
tooling/pipeline/includes/data_model/generated_models.h
View File

@@ -11,9 +11,7 @@
#include "data_model/models.h"
// ============================================================================
// Generation Output Models
// ============================================================================
/**
* @brief Generated brewery payload.
@@ -79,23 +77,39 @@ struct UserResult {
float activity_weight{};
};
// ============================================================================
// Pipeline Data Models
// ============================================================================
/**
* @brief Enriched city data with Wikipedia context.
*/
struct EnrichedCity {
Location location;
City location;
std::string region_context{};
};
/**
* @brief A physical address: the city plus a concrete postal code selected
* from that city's curated postal-code examples.
*/
struct BreweryAddress {
City city;
std::string postal_code{};
};
/**
* @brief A physical address for a generated user: the city plus a concrete
* postal code selected from that city's curated postal-code examples.
*/
struct UserAddress {
City city;
std::string postal_code{};
};
/**
* @brief Helper struct to store generated brewery data.
*/
struct BreweryRecord {
Location location;
BreweryAddress address;
BreweryResult brewery;
};
@@ -107,7 +121,7 @@ struct BreweryRecord {
* consumer can register real accounts from the pipeline's SQLite export.
*/
struct UserRecord {
Location location;
UserAddress address;
UserResult user;
std::string email{};
std::string date_of_birth{};

48
tooling/pipeline/includes/data_model/models.h
View File

@@ -21,14 +21,36 @@ class ILogger;
namespace prog_opts = boost::program_options;
// ============================================================================
// Location Models
// ============================================================================
// City Models
/**
* @brief Canonical location record for city-level generation.
* @brief Curated postal-code metadata for a city, mirroring the
* `postal_code` object in the locations dataset.
*/
struct Location {
struct PostalCodeSpec {
/**
* @brief Permissive country-wide postal-code format regex, used as the
* validation fallback for a selected example.
*/
std::string country_format_regex{};
/**
* @brief City-specific postal-code regexes (more precise than the country
* format).
*/
std::vector<std::string> city_regexes{};
/**
* @brief Concrete example postal codes for this city. The postal code
* service selects from these rather than synthesizing codes.
*/
std::vector<std::string> examples{};
};
/**
* @brief Canonical city record for city-level generation.
*/
struct City {
std::string city{};
std::string state_province{};
@@ -50,19 +72,13 @@ struct Location {
std::vector<std::string> local_languages{};
/**
* @brief Latitude in decimal degrees.
* @brief Postal-code metadata (format regex, city regexes, and concrete
* examples) sourced directly from the curated locations dataset.
*/
double latitude{};
/**
* @brief Longitude in decimal degrees.
*/
double longitude{};
PostalCodeSpec postal_code{};
};
// ============================================================================
// Name / Persona Models
// ============================================================================
/**
* @brief A sampled first/last name pair, with the source forename's gender.
@@ -131,9 +147,7 @@ struct UserPersona {
std::vector<std::string> style_affinities{};
};
// ============================================================================
// Configuration Models
// ============================================================================
/**
* @brief LLM sampling parameters.
@@ -223,9 +237,7 @@ struct ApplicationOptions {
PipelineOptions pipeline;
};
// ============================================================================
// Function Declarations
// ============================================================================
std::optional<ApplicationOptions> ParseArguments(
const int argc, char** argv, std::shared_ptr<ILogger> logger = nullptr);

2
tooling/pipeline/includes/services/curated_data/curated_data_service.h
View File

@@ -15,7 +15,7 @@
using ForenameList = std::vector<ForenameEntry>;
using SurnameList = std::vector<std::string>;
using LocationsList = std::vector<Location>;
using LocationsList = std::vector<City>;
using PersonasList = std::vector<UserPersona>;
using ForenamesByCountryMap = std::unordered_map<std::string, ForenameList>;
using SurnamesByCountryMap = std::unordered_map<std::string, SurnameList>;

16
tooling/pipeline/includes/services/database/sqlite_export_service.h
View File

@@ -45,27 +45,29 @@ class SqliteExportService final : public IExportService {
void RollbackAndCloseNoThrow() noexcept;
[[nodiscard]] std::filesystem::path BuildDatabasePath() const;
[[nodiscard]] static std::string BuildLocationKey(const Location& location);
[[nodiscard]] static std::string BuildCityKey(const City& city);
/**
* @brief Returns the row id for @p location, inserting it first if it has
* @brief Returns the row id for @p city, inserting it first if it has
* not already been seen during this run.
*
* Shared by both ProcessRecord() overloads so breweries and users
* referencing the same location resolve to the same row.
* Shared by both ProcessRecord() overloads so brewery and user addresses
* referencing the same city resolve to the same row.
*/
[[nodiscard]] sqlite3_int64 ResolveLocationId(const Location& location);
[[nodiscard]] sqlite3_int64 ResolveCityId(const City& city);
std::unique_ptr<IDateTimeProvider> date_time_provider_;
std::filesystem::path output_path_;
std::string run_timestamp_utc_;
std::filesystem::path database_path_;
SqliteDatabaseHandle db_handle_;
SqliteStatementHandle insert_location_stmt_;
SqliteStatementHandle insert_city_stmt_;
SqliteStatementHandle insert_brewery_stmt_;
SqliteStatementHandle insert_brewery_address_stmt_;
SqliteStatementHandle insert_user_stmt_;
SqliteStatementHandle insert_user_address_stmt_;
bool transaction_open_ = false;
std::unordered_map<std::string, sqlite3_int64> location_cache_;
std::unordered_map<std::string, sqlite3_int64> city_cache_;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATABASE_SQLITE_EXPORT_SERVICE_H_

122
tooling/pipeline/includes/services/database/sqlite_statement_helpers.h
View File

@@ -17,9 +17,9 @@
namespace sqlite_export_service_internal {
inline constexpr std::string_view kCreateLocationsTableSql = R"sql(
inline constexpr std::string_view kCreateCitiesTableSql = R"sql(
CREATE TABLE IF NOT EXISTS locations (
CREATE TABLE IF NOT EXISTS cities (
id INTEGER PRIMARY KEY AUTOINCREMENT,
city TEXT NOT NULL,
state_province TEXT NOT NULL,
@@ -27,9 +27,9 @@ CREATE TABLE IF NOT EXISTS locations (
country TEXT NOT NULL,
iso3166_1 TEXT NOT NULL,
local_languages_json TEXT NOT NULL,
latitude REAL NOT NULL,
longitude REAL NOT NULL,
UNIQUE(city, state_province, iso3166_2, country, latitude, longitude)
postal_code_country_format_regex TEXT NOT NULL,
postal_code_city_regex_json TEXT NOT NULL,
UNIQUE(city, state_province, iso3166_2, country)
);
)sql";
@@ -38,15 +38,29 @@ inline constexpr std::string_view kCreateBreweriesTableSql = R"sql(
CREATE TABLE IF NOT EXISTS breweries (
id INTEGER PRIMARY KEY AUTOINCREMENT,
location_id INTEGER NOT NULL,
name_en TEXT NOT NULL,
description_en TEXT NOT NULL,
name_local TEXT NOT NULL,
description_local TEXT NOT NULL,
FOREIGN KEY(location_id) REFERENCES locations(id) ON DELETE CASCADE
description_local TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_breweries_location_id ON breweries(location_id);
)sql";
inline constexpr std::string_view kCreateBreweryAddressesTableSql = R"sql(
CREATE TABLE IF NOT EXISTS brewery_addresses (
id INTEGER PRIMARY KEY AUTOINCREMENT,
brewery_id INTEGER NOT NULL,
city_id INTEGER NOT NULL,
postal_code TEXT NOT NULL,
FOREIGN KEY(brewery_id) REFERENCES breweries(id) ON DELETE CASCADE,
FOREIGN KEY(city_id) REFERENCES cities(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_brewery_addresses_brewery_id
ON brewery_addresses(brewery_id);
CREATE INDEX IF NOT EXISTS idx_brewery_addresses_city_id
ON brewery_addresses(city_id);
)sql";
@@ -54,7 +68,6 @@ inline constexpr std::string_view kCreateUsersTableSql = R"sql(
CREATE TABLE IF NOT EXISTS users (
id INTEGER PRIMARY KEY AUTOINCREMENT,
location_id INTEGER NOT NULL,
first_name TEXT NOT NULL,
last_name TEXT NOT NULL,
gender TEXT NOT NULL,
@@ -62,40 +75,61 @@ CREATE TABLE IF NOT EXISTS users (
bio TEXT NOT NULL,
activity_weight REAL NOT NULL,
email TEXT NOT NULL UNIQUE,
date_of_birth TEXT NOT NULL,
FOREIGN KEY(location_id) REFERENCES locations(id) ON DELETE CASCADE
date_of_birth TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_users_location_id ON users(location_id);
)sql";
inline constexpr std::string_view kInsertLocationSql = R"sql(
INSERT INTO locations (
inline constexpr std::string_view kCreateUserAddressesTableSql = R"sql(
CREATE TABLE IF NOT EXISTS user_addresses (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL,
city_id INTEGER NOT NULL,
postal_code TEXT NOT NULL,
FOREIGN KEY(user_id) REFERENCES users(id) ON DELETE CASCADE,
FOREIGN KEY(city_id) REFERENCES cities(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_user_addresses_user_id
ON user_addresses(user_id);
CREATE INDEX IF NOT EXISTS idx_user_addresses_city_id
ON user_addresses(city_id);
)sql";
inline constexpr std::string_view kInsertCitySql = R"sql(
INSERT INTO cities (
city,
state_province,
iso3166_2,
country,
iso3166_1,
local_languages_json,
latitude,
longitude
postal_code_country_format_regex,
postal_code_city_regex_json
) VALUES (?, ?, ?, ?, ?, ?, ?, ?);
)sql";
inline constexpr std::string_view kInsertBrewerySql = R"sql(
INSERT INTO breweries (
location_id,
name_en,
description_en,
name_local,
description_local
) VALUES (?, ?, ?, ?, ?);
) VALUES (?, ?, ?, ?);
)sql";
inline constexpr std::string_view kInsertBreweryAddressSql = R"sql(
INSERT INTO brewery_addresses (
brewery_id,
city_id,
postal_code
) VALUES (?, ?, ?);
)sql";
inline constexpr std::string_view kInsertUserSql = R"sql(
INSERT INTO users (
location_id,
first_name,
last_name,
gender,
@@ -104,33 +138,45 @@ INSERT INTO users (
activity_weight,
email,
date_of_birth
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);
) VALUES (?, ?, ?, ?, ?, ?, ?, ?);
)sql";
inline constexpr std::string_view kInsertUserAddressSql = R"sql(
INSERT INTO user_addresses (
user_id,
city_id,
postal_code
) VALUES (?, ?, ?);
)sql";
// sqlite3_bind_*() parameter indices are 1-based, matching the "?"
// placeholder order in the SQL above.
enum LocationBindIndex {
kLocationCityBindIndex = 1,
kLocationStateProvinceBindIndex,
kLocationIso31662BindIndex,
kLocationCountryBindIndex,
kLocationIso31661BindIndex,
kLocationLanguagesBindIndex,
kLocationLatitudeBindIndex,
kLocationLongitudeBindIndex,
enum CityBindIndex {
kCityCityBindIndex = 1,
kCityStateProvinceBindIndex,
kCityIso31662BindIndex,
kCityCountryBindIndex,
kCityIso31661BindIndex,
kCityLanguagesBindIndex,
kCityPostalCodeCountryFormatRegexBindIndex,
kCityPostalCodeCityRegexJsonBindIndex,
};
enum BreweryBindIndex {
kBreweryLocationIdBindIndex = 1,
kBreweryEnglishNameBindIndex,
kBreweryEnglishNameBindIndex = 1,
kBreweryEnglishDescriptionBindIndex,
kBreweryLocalNameBindIndex,
kBreweryLocalDescriptionBindIndex,
};
enum BreweryAddressBindIndex {
kBreweryAddressBreweryIdBindIndex = 1,
kBreweryAddressCityIdBindIndex,
kBreweryAddressPostalCodeBindIndex,
};
enum UserBindIndex {
kUserLocationIdBindIndex = 1,
kUserFirstNameBindIndex,
kUserFirstNameBindIndex = 1,
kUserLastNameBindIndex,
kUserGenderBindIndex,
kUserUsernameBindIndex,
@@ -140,6 +186,12 @@ enum UserBindIndex {
kUserDateOfBirthBindIndex,
};
enum UserAddressBindIndex {
kUserAddressUserIdBindIndex = 1,
kUserAddressCityIdBindIndex,
kUserAddressPostalCodeBindIndex,
};
SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle,
std::string_view sql,
const char* action);

3
tooling/pipeline/includes/services/datetime/date_time_provider.h
View File

@@ -15,6 +15,9 @@
/**
* @brief Interface for UTC timestamp providers.
*
* @todo Yagni, consider removing this and just use SystemDateTimeProvider
* directly
*/
class IDateTimeProvider {
public:

4
tooling/pipeline/includes/services/enrichment/enrichment_service.h
View File

@@ -20,10 +20,10 @@ class IEnrichmentService {
/**
* @brief Resolves contextual enrichment for a location.
*
* @param loc Location to enrich.
* @param loc City to enrich.
* @return Context text, or an empty string if unavailable.
*/
virtual std::string GetLocationContext(const Location& loc) = 0;
virtual std::string GetLocationContext(const City& loc) = 0;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ENRICHMENT_ENRICHMENT_SERVICE_H_

2
tooling/pipeline/includes/services/enrichment/mock_enrichment.h
View File

@@ -15,7 +15,7 @@
*/
class MockEnrichmentService final : public IEnrichmentService {
public:
std::string GetLocationContext(const Location& /*loc*/) override {
std::string GetLocationContext(const City& /*loc*/) override {
return {};
}
};

2
tooling/pipeline/includes/services/enrichment/wikipedia_service.h
View File

@@ -29,7 +29,7 @@ class WikipediaEnrichmentService final : public IEnrichmentService {
/**
* @brief Returns the Wikipedia-derived context for a location.
*/
[[nodiscard]] std::string GetLocationContext(const Location& loc) override;
[[nodiscard]] std::string GetLocationContext(const City& loc) override;
private:
std::string FetchExtract(std::string_view query);

2
tooling/pipeline/includes/services/logging/log_entry.h
View File

@@ -51,7 +51,7 @@ enum class PipelinePhase {
*/
Startup,
/**
* @brief Location/context enrichment (e.g. Wikipedia lookups).
* @brief City/context enrichment (e.g. Wikipedia lookups).
*/
Enrichment,
/**

79
tooling/pipeline/includes/services/postal_code/mock_postal_code_service.h Normal file
View File

@@ -0,0 +1,79 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_
/**
* @file services/postal_code/mock_postal_code_service.h
* @brief Data-driven postal code service that selects the first curated
* example for a city and validates it against the city's postal-code regex.
*/
#include <format>
#include <regex>
#include <stdexcept>
#include <string>
#include "services/postal_code/postal_code_service.h"
/**
* @brief Postal code service backed entirely by the curated locations
* dataset: it returns the city's first example postal code after checking it
* against the curated postal-code regex.
*
* A precise city regex is preferred when present; otherwise the permissive
* country format regex is used as the fallback. A city with no examples, or an
* example that fails validation, throws so the caller can skip the city.
*/
class MockPostalCodeService final : public IPostalCodeService {
public:
std::string GeneratePostalCode(const City& city) override {
const PostalCodeSpec& spec = city.postal_code;
if (spec.examples.empty()) {
throw std::runtime_error(std::format(
"No postal-code examples available for city '{}'", city.city));
}
const std::string& candidate = spec.examples.front();
if (!MatchesAnyPattern(candidate, spec)) {
throw std::runtime_error(
std::format("Postal code '{}' for city '{}' does not match the "
"curated postal-code regex",
candidate, city.city));
}
return candidate;
}
private:
/**
* @brief Returns true if @p candidate matches any city regex, or the
* country format regex when no city regex matches. When the spec carries no
* usable regex at all, validation is skipped (returns true).
*/
static bool MatchesAnyPattern(const std::string& candidate,
const PostalCodeSpec& spec) {
bool has_pattern = false;
for (const std::string& pattern : spec.city_regexes) {
if (pattern.empty()) {
continue;
}
has_pattern = true;
if (std::regex_match(candidate, std::regex(pattern))) {
return true;
}
}
if (!spec.country_format_regex.empty()) {
has_pattern = true;
if (std::regex_match(candidate, std::regex(spec.country_format_regex))) {
return true;
}
}
return !has_pattern;
}
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_

35
tooling/pipeline/includes/services/postal_code/postal_code_service.h Normal file
View File

@@ -0,0 +1,35 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_
/**
* @file services/postal_code/postal_code_service.h
* @brief Abstraction for resolving a postal code for a city.
*/
#include <string>
#include "data_model/models.h"
/**
* @brief Interface for services that resolve a postal code for a city.
*/
class IPostalCodeService {
public:
IPostalCodeService() = default;
virtual ~IPostalCodeService() = default;
IPostalCodeService(const IPostalCodeService&) = delete;
IPostalCodeService& operator=(const IPostalCodeService&) = delete;
IPostalCodeService(IPostalCodeService&&) = delete;
IPostalCodeService& operator=(IPostalCodeService&&) = delete;
/**
* @brief Resolves a postal code for @p city.
*
* @param city City whose curated postal-code examples supply the result.
* @return A postal code string, or an empty string if none is available.
*/
virtual std::string GeneratePostalCode(const City& city) = 0;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_

32
tooling/pipeline/includes/services/postal_code/xeger_postal_code_service.h Normal file
View File

@@ -0,0 +1,32 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_XEGER_POSTAL_CODE_SERVICE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_XEGER_POSTAL_CODE_SERVICE_H_
/**
* @file services/postal_code/xeger_postal_code_service.h
* @brief Postal code service that generates a fresh, format-conformant code
* per call from the curated regex patterns, instead of returning a fixed
* example.
*/
#include <random>
#include "services/postal_code/postal_code_service.h"
/**
* @brief Postal code service using regex-driven generation (xeger).
*
* Picks among the city's curated `city_regexes` (falls back to
* `country_format_regex`), then generates a random string matching that pattern
* via GenerateStringFromRegex().
*/
class XegerPostalCodeService final : public IPostalCodeService {
public:
XegerPostalCodeService();
std::string GeneratePostalCode(const City& city) override;
private:
std::mt19937 rng_;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_XEGER_POSTAL_CODE_SERVICE_H_

38
tooling/pipeline/includes/services/xeger/xeger.h Normal file
View File

@@ -0,0 +1,38 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_XEGER_XEGER_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_XEGER_XEGER_H_
#include <random>
#include <string>
#include <string_view>
/**
* @brief Generates a random string matching @p pattern.
*
* Supported syntax:
* - literals
* - `.`
* - alternation `|`
* - groups `(...)` and non-capturing groups `(?:...)`
* - quantifiers `*` `+` `?` `{n}` `{n,}` `{n,m}`
* - character classes `[...]` / `[^...]`
* - shorthand escapes `\d \D \w \W \s \S \n \t \r`
*
* Not supported: lookaround, backreferences, and lazy quantifiers.
*
* @param pattern Regular expression to generate a matching
* string for.
* @param rng Random source driving alternation/quantifier/
* class choices.
* @param unbounded_repeat_cap Extra repeats allowed on top of the declared
* minimum for unbounded quantifiers
* (`*`, `+`, `{n,}`).
*
* @return A string satisfying `std::regex_match(result, std::regex(pattern))`
* for any @p pattern within the supported syntax subset.
*
* @throws std::runtime_error if @p pattern cannot be parsed.
*/
std::string GenerateStringFromRegex(std::string_view pattern, std::mt19937& rng,
int unbounded_repeat_cap = 8);
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_XEGER_XEGER_H_

201
tooling/pipeline/includes/services/xeger/xeger_internal.h Normal file
View File

@@ -0,0 +1,201 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_XEGER_XEGER_INTERNAL_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_XEGER_XEGER_INTERNAL_H_
/**
* @file services/xeger/xeger_internal.h
* @brief Declarations shared by the xeger parser and generator.
*
* 1. Parser Turns the pattern text into an AST (Literal, AnyChar,
* CharClass, Concat, Alternation, Repeat nodes).
* 2. Visitor Double-dispatch contract -- each AST-walking
* operation implements this.
* 3. XegerGenerator a Visitor that emits a random character wherever the
* AST says a character belongs, respecting alternation
* choices and repeat counts.
*/
#include <memory>
#include <optional>
#include <random>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
namespace xeger_internal {
// Regex syntax characters
constexpr char kAnchorStart = '^';
constexpr char kAnchorEnd = '$';
constexpr char kAlternation = '|';
constexpr char kGroupOpen = '(';
constexpr char kGroupClose = ')';
constexpr char kNonCapturingMarker = '?';
constexpr char kNonCapturingColon = ':';
constexpr char kClassOpen = '[';
constexpr char kClassClose = ']';
constexpr char kClassNegate = '^';
constexpr char kClassRange = '-';
constexpr char kAnyChar = '.';
constexpr char kEscape = '\\';
constexpr char kStar = '*';
constexpr char kPlus = '+';
constexpr char kOptional = '?';
constexpr char kBraceOpen = '{';
constexpr char kBraceClose = '}';
constexpr char kBraceComma = ',';
// AST
struct Literal;
struct AnyChar;
struct CharClass;
struct Concat;
struct Alternation;
struct Repeat;
struct Visitor {
virtual void Visit(const Literal&) = 0;
virtual void Visit(const AnyChar&) = 0;
virtual void Visit(const CharClass&) = 0;
virtual void Visit(const Concat&) = 0;
virtual void Visit(const Alternation&) = 0;
virtual void Visit(const Repeat&) = 0;
virtual ~Visitor() = default;
};
struct Node {
virtual void Accept(Visitor&) const = 0;
virtual ~Node() = default;
};
using NodePtr = std::unique_ptr<Node>;
template <typename Derived>
struct Visitable : Node {
void Accept(Visitor& visitor) const override {
visitor.Visit(static_cast<const Derived&>(*this));
}
};
// One fixed character, e.g. 'a'.
struct Literal : Visitable<Literal> {
char ch;
explicit Literal(const char c) : ch(c) {}
};
// The "." wildcard: any single printable character.
struct AnyChar : Visitable<AnyChar> {};
// A set of allowed characters, stored as inclusive ranges. A single
// character 'x' is the range {'x', 'x'}. `negated` flips the meaning to
// "any character NOT listed" (i.e. `[^...]`).
struct CharClass : Visitable<CharClass> {
std::vector<std::pair<char, char>> ranges;
bool negated = false;
};
// "Do these children one after another."
struct Concat : Visitable<Concat> {
std::vector<NodePtr> parts;
};
// "Pick exactly one of these options at random" -- the `|` operator.
struct Alternation : Visitable<Alternation> {
std::vector<NodePtr> options;
};
// "Repeat one child between min and max times."
// a* -> min 0, max infinite
// a+ -> min 1, max infinite
// a? -> min 0, max 1
// a{2,5} -> min 2, max 5
struct Repeat : Visitable<Repeat> {
NodePtr child;
unsigned int min = 0;
std::optional<unsigned int> max; // std::nullopt means "no upper bound"
};
// Parser (recursive descent)
//
// Precedence, loosest to tightest binding:
// ParseAlternation ::= a|b|c
// ParseConcat ::= abc
// ParseRepeat ::= a* a{2,5}
// ParseAtom ::= a . [..] (..) \d
class Parser {
public:
explicit Parser(std::string_view pattern);
NodePtr Parse();
private:
std::string src_;
size_t pos_ = 0;
// Strips anchors (leading '^' and trailing '$') as they are not needed for
// generation.
static std::string StripAnchors(std::string_view pattern);
[[nodiscard]] bool AtEnd() const;
[[nodiscard]] char Peek() const;
char Advance();
bool Match(char c);
void Expect(char c);
// Level 1: alternation -- ( concat | concat | concat )
NodePtr ParseAlternation();
// Level 2: concatenation -- a run of items back to back
NodePtr ParseConcat();
// Level 3: an atom plus an optional quantifier
NodePtr ParseRepeat();
// Curly-brace quantifiers: {n}, {n,}, {n,m}
NodePtr ParseBrace(NodePtr atom);
// Level 4: an atom -- the smallest standalone unit
NodePtr ParseAtom();
static NodePtr MakeClass(std::vector<std::pair<char, char>> ranges,
bool negated);
NodePtr ParseEscape();
NodePtr ParseCharClass();
// Reads one character inside [ ], turning \] \n etc. into the real
// character they represent.
char ClassChar();
};
// Generator -- a Visitor that emits a random matching string
class XegerGenerator final : public Visitor {
public:
XegerGenerator(std::mt19937& rng, int unbounded_cap);
std::string Run(const Node& root);
void Visit(const Literal& node) override;
void Visit(const AnyChar&) override;
void Visit(const CharClass& node) override;
void Visit(const Concat& node) override;
void Visit(const Alternation& node) override;
void Visit(const Repeat& node) override;
private:
std::mt19937& rng_;
int unbounded_cap_;
std::string out_;
static bool InRanges(const CharClass& node, char c);
char PickFromClass(const CharClass& node);
};
} // namespace xeger_internal
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_XEGER_XEGER_INTERNAL_H_

1108
tooling/pipeline/locations.json
View File

File diff suppressed because it is too large Load Diff

2
tooling/pipeline/src/biergarten_pipeline_orchestrator/biergarten_pipeline_orchestrator.cc
View File

@@ -13,10 +13,12 @@ BiergartenPipelineOrchestrator::BiergartenPipelineOrchestrator(
std::unique_ptr<DataGenerator> generator,
std::unique_ptr<IExportService> exporter,
std::unique_ptr<ICuratedDataService> curated_data_service,
std::unique_ptr<IPostalCodeService> postal_code_service,
const ApplicationOptions& application_options)
: logger_(std::move(logger)),
context_service_(std::move(context_service)),
generator_(std::move(generator)),
exporter_(std::move(exporter)),
curated_data_service_(std::move(curated_data_service)),
postal_code_service_(std::move(postal_code_service)),
application_options_(application_options) {}

35
tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_breweries.cc
View File

@@ -22,12 +22,15 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
const auto generate_record =
[this, &skipped_count](
const Location& location,
const std::string& region_context) -> std::optional<BreweryRecord> {
const EnrichedCity& enriched_city) -> std::optional<BreweryRecord> {
try {
const BreweryResult brewery =
generator_->GenerateBrewery(location, region_context);
return BreweryRecord{.location = location, .brewery = brewery};
const BreweryResult brewery = generator_->GenerateBrewery(enriched_city);
const std::string postal_code =
postal_code_service_->GeneratePostalCode(enriched_city.location);
return BreweryRecord{
.address = BreweryAddress{.city = enriched_city.location,
.postal_code = postal_code},
.brewery = brewery};
} catch (const std::exception& e) {
++skipped_count;
@@ -36,7 +39,8 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format("[Pipeline] Skipping city '{}' ({}): brewery "
"generation failed: {}",
location.city, location.country, e.what())});
enriched_city.location.city,
enriched_city.location.country, e.what())});
return std::nullopt;
}
};
@@ -47,19 +51,18 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
exporter_->ProcessRecord(record);
} catch (const std::exception& export_exception) {
++export_failed_count;
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format("[Pipeline] Generated brewery for '{}' ({}) "
"but SQLite export failed: {}",
record.location.city, record.location.country,
export_exception.what())});
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"[Pipeline] Generated brewery for '{}' ({}) "
"but SQLite export failed: {}",
record.address.city.city, record.address.city.country,
export_exception.what())});
}
};
for (const auto& [location, region_context] : cities) {
const std::optional<BreweryRecord> record =
generate_record(location, region_context);
for (const EnrichedCity& enriched_city : cities) {
const std::optional<BreweryRecord> record = generate_record(enriched_city);
if (!record.has_value()) {
continue;
}

19
tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc
View File

@@ -135,9 +135,12 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
try {
const UserResult user =
generator_->GenerateUser(city, persona, sampled_name);
const std::string postal_code =
postal_code_service_->GeneratePostalCode(city.location);
return UserRecord{
.location = city.location,
.address =
UserAddress{.city = city.location, .postal_code = postal_code},
.user = user,
.email = BuildEmail(sampled_name, used_email_local_parts),
.date_of_birth = GenerateDateOfBirth(rng),
@@ -161,13 +164,13 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
} catch (const std::exception& export_exception) {
++export_failed_count;
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format("[Pipeline] Generated user for '{}' ({}) but "
"SQLite export failed: {}",
record.location.city, record.location.country,
export_exception.what())});
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Generated user for '{}' ({}) but "
"SQLite export failed: {}",
record.address.city.city, record.address.city.country,
export_exception.what())});
}
};

18
tooling/pipeline/src/biergarten_pipeline_orchestrator/log_results.cc
View File

@@ -14,20 +14,20 @@
void BiergartenPipelineOrchestrator::LogResults() const {
boost::json::array brewery_output;
for (const auto& [location, brewery] : generated_breweries_) {
for (const auto& [address, brewery] : generated_breweries_) {
const City& city = address.city;
brewery_output.push_back(boost::json::object{
{"name_en", brewery.name_en},
{"description_en", brewery.description_en},
{"name_local", brewery.name_local},
{"description_local", brewery.description_local},
{"location", boost::json::object{
{"city", location.city},
{"country", location.country},
{"state_province", location.state_province},
{"iso3166_2", location.iso3166_2},
{"latitude", location.latitude},
{"longitude", location.longitude},
}}});
{"address", boost::json::object{
{"city", city.city},
{"country", city.country},
{"state_province", city.state_province},
{"iso3166_2", city.iso3166_2},
{"postal_code", address.postal_code},
}}});
}
std::ostringstream brewery_oss;

8
tooling/pipeline/src/biergarten_pipeline_orchestrator/query_cities_with_countries.cc
View File

@@ -14,13 +14,13 @@
#include "services/curated_data/curated_json_data_service.h"
#include "services/logging/logger.h"
std::vector<Location>
BiergartenPipelineOrchestrator::QueryCitiesWithCountries() {
std::vector<City>
BiergartenPipelineOrchestrator::QueryLocations() {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "=== GEOGRAPHIC DATA OVERVIEW ==="});
const std::vector<Location>& all_locations =
const std::vector<City>& all_locations =
curated_data_service_->LoadLocations();
const size_t sample_count = std::min(
@@ -31,7 +31,7 @@ BiergartenPipelineOrchestrator::QueryCitiesWithCountries() {
static_cast<std::iter_difference_t<decltype(all_locations.cbegin())>>(
sample_count);
std::vector<Location> sampled_locations;
std::vector<City> sampled_locations;
sampled_locations.reserve(sample_count);
std::random_device random_generator;

2
tooling/pipeline/src/biergarten_pipeline_orchestrator/run.cc
View File

@@ -14,7 +14,7 @@ bool BiergartenPipelineOrchestrator::Run() {
try {
exporter_->Initialize();
std::vector<Location> cities = QueryCitiesWithCountries();
std::vector<City> cities = QueryLocations();
std::vector<EnrichedCity> enriched;
enriched.reserve(cities.size());

6
tooling/pipeline/src/data_generation/llama/generate_brewery.cc
View File

@@ -36,7 +36,9 @@ static std::string FormatLocalLanguageCodes(
static constexpr int kBreweryInitialMaxTokens = 2800;
BreweryResult LlamaGenerator::GenerateBrewery(
const Location& location, const std::string& region_context) {
const EnrichedCity& enriched_city) {
const City& location = enriched_city.location;
const std::string& region_context = enriched_city.region_context;
/**
* Preprocess and truncate region context to manageable size
*/
@@ -66,7 +68,7 @@ BreweryResult LlamaGenerator::GenerateBrewery(
* Store location context for retry prompts (without repeating full context)
*/
const std::string retry_location =
std::format("Location: {}{}\nLocal language codes: {}", location.city,
std::format("City: {}{}\nLocal language codes: {}", location.city,
country_suffix, local_language_codes);
/**

4
tooling/pipeline/src/data_generation/mock/deterministic_hash.cc
View File

@@ -8,14 +8,14 @@
#include "data_generation/mock_generator.h"
size_t MockGenerator::DeterministicHash(const Location& location) {
size_t MockGenerator::DeterministicHash(const City& location) {
size_t seed = 0;
boost::hash_combine(seed, location.city);
boost::hash_combine(seed, location.country);
return seed;
}
size_t MockGenerator::DeterministicHash(const Location& location,
size_t MockGenerator::DeterministicHash(const City& location,
const UserPersona& persona,
const Name& name) {
size_t seed = DeterministicHash(location);

3
tooling/pipeline/src/data_generation/mock/generate_brewery.cc
View File

@@ -11,7 +11,8 @@
#include "data_generation/mock_generator.h"
BreweryResult MockGenerator::GenerateBrewery(
const Location& location, const std::string& /*region_context*/) {
const EnrichedCity& enriched_city) {
const City& location = enriched_city.location;
const size_t hash = DeterministicHash(location);
const std::string_view adjective =

18
tooling/pipeline/src/main.cc
View File

@@ -86,6 +86,24 @@ int main(const int argc, char** argv) {
di::bind<ApplicationOptions>().to(options),
di::bind<std::string>().to(model_path),
di::bind<IExportService>().to<SqliteExportService>(),
di::bind<IPostalCodeService>().to(
[options, &log_producer]() -> std::unique_ptr<IPostalCodeService> {
if (options.generator.use_mocked) {
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Postal code: mock (curated examples)"});
return std::make_unique<MockPostalCodeService>();
}
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Postal code: Xeger (regex-based generation)"});
return std::make_unique<XegerPostalCodeService>();
}),
di::bind<ICuratedDataService>().to(
[options, &log_producer]() -> std::unique_ptr<ICuratedDataService> {
if (options.generator.use_mocked) {

32
tooling/pipeline/src/services/curated_data/curated_json_data_service.cc
View File

@@ -1,6 +1,6 @@
/**
* @file json_handling/json_loader.cc
* @brief Parses curated location JSON input into strongly typed Location
* @brief Parses curated location JSON input into strongly typed City
* records with strict field validation and descriptive error reporting.
*/
@@ -30,16 +30,6 @@ static std::string ReadRequiredString(const boost::json::object& object,
return std::string(text);
}
static double ReadRequiredNumber(const boost::json::object& object,
const char* key) {
const boost::json::value* value = object.if_contains(key);
if (value == nullptr || !value->is_number()) {
throw std::runtime_error(
std::format("Missing or invalid numeric field: {}", key));
}
return value->to_number<double>();
}
static std::vector<std::string> ReadRequiredStringArray(
const boost::json::object& object, const char* key) {
const boost::json::value* value = object.if_contains(key);
@@ -61,6 +51,21 @@ static std::vector<std::string> ReadRequiredStringArray(
return items;
}
static PostalCodeSpec ReadPostalCodeSpec(const boost::json::object& object) {
const boost::json::value* value = object.if_contains("postal_code");
if (value == nullptr || !value->is_object()) {
throw std::runtime_error("Missing or invalid object field: postal_code");
}
const auto& postal_code = value->as_object();
return PostalCodeSpec{
.country_format_regex =
ReadRequiredString(postal_code, "country_format_regex"),
.city_regexes = ReadRequiredStringArray(postal_code, "city_regex"),
.examples = ReadRequiredStringArray(postal_code, "examples"),
};
}
namespace {
boost::json::value ParseJsonFile(const std::filesystem::path& filepath,
@@ -133,15 +138,14 @@ const LocationsList& CuratedJsonDataService::LoadLocations() {
}
const auto& object = item.as_object();
locations.push_back(Location{
locations.push_back(City{
.city = ReadRequiredString(object, "city"),
.state_province = ReadRequiredString(object, "state_province"),
.iso3166_2 = ReadRequiredString(object, "iso3166_2"),
.country = ReadRequiredString(object, "country"),
.iso3166_1 = ReadRequiredString(object, "iso3166_1"),
.local_languages = ReadRequiredStringArray(object, "local_languages"),
.latitude = ReadRequiredNumber(object, "latitude"),
.longitude = ReadRequiredNumber(object, "longitude"),
.postal_code = ReadPostalCodeSpec(object),
});
}
cache_.locations = std::move(locations);

72
tooling/pipeline/src/services/curated_data/mock_curated_data_service.cc
View File

@@ -7,38 +7,46 @@
MockCuratedDataService::MockCuratedDataService()
: locations_{
Location{.city = "Portland",
.state_province = "Oregon",
.iso3166_2 = "US-OR",
.country = "United States",
.iso3166_1 = "US",
.local_languages = {"en"},
.latitude = 45.5152,
.longitude = -122.6784},
Location{.city = "Munich",
.state_province = "Bavaria",
.iso3166_2 = "DE-BY",
.country = "Germany",
.iso3166_1 = "DE",
.local_languages = {"de"},
.latitude = 48.1351,
.longitude = 11.5820},
Location{.city = "Lyon",
.state_province = "Auvergne-Rhone-Alpes",
.iso3166_2 = "FR-ARA",
.country = "France",
.iso3166_1 = "FR",
.local_languages = {"fr"},
.latitude = 45.7640,
.longitude = 4.8357},
Location{.city = "Brussels",
.state_province = "Brussels-Capital",
.iso3166_2 = "BE-BRU",
.country = "Belgium",
.iso3166_1 = "BE",
.local_languages = {"nl", "fr"},
.latitude = 50.8503,
.longitude = 4.3517},
City{.city = "Portland",
.state_province = "Oregon",
.iso3166_2 = "US-OR",
.country = "United States",
.iso3166_1 = "US",
.local_languages = {"en"},
.postal_code =
PostalCodeSpec{.country_format_regex = R"(^\d{5}$)",
.city_regexes = {R"(^972\d{2}$)"},
.examples = {"97201", "97202", "97209"}}},
City{.city = "Munich",
.state_province = "Bavaria",
.iso3166_2 = "DE-BY",
.country = "Germany",
.iso3166_1 = "DE",
.local_languages = {"de"},
.postal_code =
PostalCodeSpec{.country_format_regex = R"(^\d{5}$)",
.city_regexes = {R"(^8\d{4}$)"},
.examples = {"80331", "80333", "81667"}}},
City{.city = "Lyon",
.state_province = "Auvergne-Rhone-Alpes",
.iso3166_2 = "FR-ARA",
.country = "France",
.iso3166_1 = "FR",
.local_languages = {"fr"},
.postal_code =
PostalCodeSpec{.country_format_regex = R"(^\d{5}$)",
.city_regexes = {R"(^690\d{2}$)"},
.examples = {"69001", "69002", "69007"}}},
City{.city = "Brussels",
.state_province = "Brussels-Capital",
.iso3166_2 = "BE-BRU",
.country = "Belgium",
.iso3166_1 = "BE",
.local_languages = {"nl", "fr"},
.postal_code =
PostalCodeSpec{.country_format_regex = R"(^\d{4}$)",
.city_regexes = {R"(^1\d{3}$)"},
.examples = {"1000", "1050", "1180"}}},
},
personas_{
UserPersona{.name = "Hophead Explorer",

2
tooling/pipeline/src/services/enrichment/wikipedia/get_summary.cc
View File

@@ -11,7 +11,7 @@
#include "services/enrichment/wikipedia_service.h"
std::string WikipediaEnrichmentService::GetLocationContext(
const Location& loc) {
const City& loc) {
using namespace std::literals::chrono_literals;
if (!this->client_) {
if (logger_) {

44
tooling/pipeline/src/services/postal_code/xeger_postal_code_service.cc Normal file
View File

@@ -0,0 +1,44 @@
/**
* @file services/postal_code/xeger_postal_code_service.cc
* @brief XegerPostalCodeService::GeneratePostalCode() implementation.
*/
#include "services/postal_code/xeger_postal_code_service.h"
#include <format>
#include <random>
#include <stdexcept>
#include <string>
#include <string_view>
#include <vector>
#include "services/xeger/xeger.h"
XegerPostalCodeService::XegerPostalCodeService()
: rng_(std::random_device{}()) {}
std::string XegerPostalCodeService::GeneratePostalCode(const City& city) {
const PostalCodeSpec& spec = city.postal_code;
std::vector<std::string_view> city_patterns;
for (const std::string& pattern : spec.city_regexes) {
if (!pattern.empty()) {
city_patterns.push_back(pattern);
}
}
std::string_view pattern;
if (!city_patterns.empty()) {
std::uniform_int_distribution<size_t> dist(0, city_patterns.size() - 1);
pattern = city_patterns[dist(rng_)];
} else if (!spec.country_format_regex.empty()) {
pattern = spec.country_format_regex;
} else {
throw std::runtime_error(std::format(
"XegerPostalCodeService: no postal-code pattern available for "
"city '{}'",
city.city));
}
return GenerateStringFromRegex(pattern, rng_);
}

11
tooling/pipeline/src/services/prompt_directory.cc
View File

@@ -15,9 +15,7 @@
#include <string_view>
#include <utility>
// ---------------------------------------------------------------------------
// PromptDirectory
// ---------------------------------------------------------------------------
PromptDirectory::PromptDirectory(const std::filesystem::path& prompt_dir)
: PromptDirectory(prompt_dir, nullptr) {}
@@ -27,21 +25,21 @@ PromptDirectory::PromptDirectory(const std::filesystem::path& prompt_dir,
: prompt_dir_(prompt_dir), logger_(std::move(logger)) {
std::error_code ec;
// Scenario 4: directory must exist.
// Directory must exist.
if (!std::filesystem::exists(prompt_dir_, ec) || ec) {
throw std::runtime_error(
"PromptDirectory: prompt directory does not exist: " +
prompt_dir_.string());
}
// Scenario 4: path must be a directory, not a file.
// Path must be a directory, not a file.
if (!std::filesystem::is_directory(prompt_dir_, ec) || ec) {
throw std::runtime_error(
"PromptDirectory: prompt directory path is not a directory: " +
prompt_dir_.string());
}
// Scenario 4: directory must be readable (probe with directory_iterator).
// Directory must be readable (probe with directory_iterator).
std::filesystem::directory_iterator probe(prompt_dir_, ec);
if (ec) {
throw std::runtime_error(std::format(
@@ -68,7 +66,7 @@ std::string PromptDirectory::Load(std::string_view key) {
return cache_it->second;
}
// Scenario 3: resolve <prompt_dir>/<key>.md and require it to exist.
// Resolve <prompt_dir>/<key>.md and require it to exist.
const std::filesystem::path file_path =
prompt_dir_ / std::filesystem::path(std::format("{}.md", key_str));
@@ -98,6 +96,7 @@ std::string PromptDirectory::Load(std::string_view key) {
key_str, file_path.string(), content.size())});
}
// Cache the loaded content for future calls.
cache_.emplace(key_str, content);
return content;
}

6
tooling/pipeline/src/services/sqlite/finalize.cc
View File

@@ -14,9 +14,11 @@ void SqliteExportService::Finalize() {
}
try {
insert_user_address_stmt_.reset();
insert_user_stmt_.reset();
insert_brewery_address_stmt_.reset();
insert_brewery_stmt_.reset();
insert_location_stmt_.reset();
insert_city_stmt_.reset();
if (transaction_open_) {
sqlite_export_service_internal::ExecSql(
db_handle_, "COMMIT;", "Failed to commit SQLite transaction");
@@ -24,7 +26,7 @@ void SqliteExportService::Finalize() {
}
db_handle_.reset();
location_cache_.clear();
city_cache_.clear();
} catch (...) {
RollbackAndCloseNoThrow();
throw;

3
tooling/pipeline/src/services/sqlite/helpers/sqlite_statement_helpers.cc
View File

@@ -88,7 +88,8 @@ sqlite3_int64 LastInsertRowId(const SqliteDatabaseHandle& db_handle) {
}
std::string SerializeVector(const std::vector<std::string>& str_vec) {
boost::json::array array(str_vec.size());
boost::json::array array;
array.reserve(str_vec.size());
for (const auto& s : str_vec) {
array.emplace_back(s);
}

31
tooling/pipeline/src/services/sqlite/initialize.cc
View File

@@ -10,7 +10,6 @@
#include <string>
#include "services/database/sqlite_export_service.h"
#include "services/database/sqlite_export_service_helpers.h"
std::filesystem::path SqliteExportService::BuildDatabasePath() const {
std::filesystem::path base_filename("biergarten_seed_" + run_timestamp_utc_ +
@@ -28,26 +27,40 @@ std::filesystem::path SqliteExportService::BuildDatabasePath() const {
void SqliteExportService::InitializeSchema() const {
sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateLocationsTableSql,
"Failed to create SQLite locations table");
db_handle_, sqlite_export_service_internal::kCreateCitiesTableSql,
"Failed to create SQLite cities table");
sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateBreweriesTableSql,
"Failed to create SQLite breweries table");
sqlite_export_service_internal::ExecSql(
db_handle_,
sqlite_export_service_internal::kCreateBreweryAddressesTableSql,
"Failed to create SQLite brewery_addresses table");
sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateUsersTableSql,
"Failed to create SQLite users table");
sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateUserAddressesTableSql,
"Failed to create SQLite user_addresses table");
}
void SqliteExportService::PrepareStatements() {
insert_location_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertLocationSql,
"Failed to prepare SQLite location insert statement");
insert_city_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertCitySql,
"Failed to prepare SQLite city insert statement");
insert_brewery_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertBrewerySql,
"Failed to prepare SQLite brewery insert statement");
insert_brewery_address_stmt_ =
sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertBreweryAddressSql,
"Failed to prepare SQLite brewery address insert statement");
insert_user_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertUserSql,
"Failed to prepare SQLite user insert statement");
insert_user_address_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertUserAddressSql,
"Failed to prepare SQLite user address insert statement");
}
void SqliteExportService::RollbackAndCloseNoThrow() noexcept {
@@ -60,11 +73,13 @@ void SqliteExportService::RollbackAndCloseNoThrow() noexcept {
transaction_open_ = false;
}
insert_user_address_stmt_.reset();
insert_user_stmt_.reset();
insert_brewery_address_stmt_.reset();
insert_brewery_stmt_.reset();
insert_location_stmt_.reset();
insert_city_stmt_.reset();
db_handle_.reset();
location_cache_.clear();
city_cache_.clear();
}
void SqliteExportService::Initialize() {

160
tooling/pipeline/src/services/sqlite/process_record.cc
View File

@@ -1,10 +1,9 @@
/**
* @file services/sqlite/process_record.cc
* @brief SqliteExportService::ProcessRecord() implementation
* and the shared location-resolution helper.
* and the shared city-resolution helper.
*/
#include <iomanip>
#include <sstream>
#include <stdexcept>
#include <string>
@@ -12,94 +11,91 @@
#include "services/database/sqlite_export_service.h"
#include "services/database/sqlite_export_service_helpers.h"
constexpr int kLocationPrecision = 17;
std::string SqliteExportService::BuildLocationKey(const Location& location) {
std::string SqliteExportService::BuildCityKey(const City& city) {
std::ostringstream key_stream;
key_stream << location.city << '\n'
<< location.state_province << '\n'
<< location.iso3166_2 << '\n'
<< location.country << '\n'
<< location.iso3166_1 << '\n'
<< std::setprecision(kLocationPrecision) << location.latitude
<< '\n'
<< std::setprecision(kLocationPrecision) << location.longitude
<< '\n'
key_stream << city.city << '\n'
<< city.state_province << '\n'
<< city.iso3166_2 << '\n'
<< city.country << '\n'
<< city.iso3166_1 << '\n'
<< sqlite_export_service_internal::SerializeVector(
location.local_languages);
city.local_languages);
return key_stream.str();
}
sqlite3_int64 SqliteExportService::ResolveLocationId(const Location& location) {
const std::string location_key = BuildLocationKey(location);
const auto cached_location = location_cache_.find(location_key);
if (cached_location != location_cache_.end()) {
return cached_location->second;
sqlite3_int64 SqliteExportService::ResolveCityId(const City& city) {
const std::string city_key = BuildCityKey(city);
const auto cached_city = city_cache_.find(city_key);
if (cached_city != city_cache_.end()) {
return cached_city->second;
}
const std::string local_languages_json =
sqlite_export_service_internal::SerializeVector(location.local_languages);
sqlite_export_service_internal::SerializeVector(city.local_languages);
const std::string city_regex_json =
sqlite_export_service_internal::SerializeVector(
city.postal_code.city_regexes);
sqlite_export_service_internal::Bind(
insert_location_stmt_,
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kLocationCityBindIndex,
.value = location.city,
.action = "Failed to bind SQLite location city"});
.index = sqlite_export_service_internal::kCityCityBindIndex,
.value = city.city,
.action = "Failed to bind SQLite city name"});
sqlite_export_service_internal::Bind(
insert_location_stmt_,
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index =
sqlite_export_service_internal::kLocationStateProvinceBindIndex,
.value = location.state_province,
.action = "Failed to bind SQLite location state/province"});
.index = sqlite_export_service_internal::kCityStateProvinceBindIndex,
.value = city.state_province,
.action = "Failed to bind SQLite city state/province"});
sqlite_export_service_internal::Bind(
insert_location_stmt_,
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kLocationIso31662BindIndex,
.value = location.iso3166_2,
.action = "Failed to bind SQLite location ISO 3166-2 code"});
.index = sqlite_export_service_internal::kCityIso31662BindIndex,
.value = city.iso3166_2,
.action = "Failed to bind SQLite city ISO 3166-2 code"});
sqlite_export_service_internal::Bind(
insert_location_stmt_,
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kLocationCountryBindIndex,
.value = location.country,
.action = "Failed to bind SQLite location country"});
.index = sqlite_export_service_internal::kCityCountryBindIndex,
.value = city.country,
.action = "Failed to bind SQLite city country"});
sqlite_export_service_internal::Bind(
insert_location_stmt_,
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kLocationIso31661BindIndex,
.value = location.iso3166_1,
.action = "Failed to bind SQLite location ISO 3166-1 code"});
.index = sqlite_export_service_internal::kCityIso31661BindIndex,
.value = city.iso3166_1,
.action = "Failed to bind SQLite city ISO 3166-1 code"});
sqlite_export_service_internal::Bind(
insert_location_stmt_,
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kLocationLanguagesBindIndex,
.index = sqlite_export_service_internal::kCityLanguagesBindIndex,
.value = local_languages_json,
.action = "Failed to bind SQLite location languages"});
.action = "Failed to bind SQLite city languages"});
sqlite_export_service_internal::Bind(
insert_location_stmt_,
sqlite_export_service_internal::BoundParam{
.index = sqlite_export_service_internal::kLocationLatitudeBindIndex,
.value = location.latitude,
.action = "Failed to bind SQLite location latitude"});
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::
kCityPostalCodeCountryFormatRegexBindIndex,
.value = city.postal_code.country_format_regex,
.action = "Failed to bind SQLite city postal-code format regex"});
sqlite_export_service_internal::Bind(
insert_location_stmt_,
sqlite_export_service_internal::BoundParam{
.index = sqlite_export_service_internal::kLocationLongitudeBindIndex,
.value = location.longitude,
.action = "Failed to bind SQLite location longitude"});
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::
kCityPostalCodeCityRegexJsonBindIndex,
.value = city_regex_json,
.action = "Failed to bind SQLite city postal-code city regexes"});
sqlite_export_service_internal::StepStatement(
db_handle_, insert_location_stmt_,
"Failed to insert SQLite location row");
db_handle_, insert_city_stmt_, "Failed to insert SQLite city row");
const sqlite3_int64 location_id =
const sqlite3_int64 city_id =
sqlite_export_service_internal::LastInsertRowId(db_handle_);
location_cache_.emplace(location_key, location_id);
sqlite_export_service_internal::ResetStatement(insert_location_stmt_);
city_cache_.emplace(city_key, city_id);
sqlite_export_service_internal::ResetStatement(insert_city_stmt_);
return location_id;
return city_id;
}
uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
@@ -107,14 +103,7 @@ uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
throw std::runtime_error("SQLite export service is not initialized");
}
const sqlite3_int64 location_id = ResolveLocationId(brewery.location);
sqlite_export_service_internal::Bind(
insert_brewery_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index = sqlite_export_service_internal::kBreweryLocationIdBindIndex,
.value = location_id,
.action = "Failed to bind SQLite brewery location id"});
const sqlite3_int64 city_id = ResolveCityId(brewery.address.city);
sqlite_export_service_internal::Bind(
insert_brewery_stmt_,
@@ -149,7 +138,38 @@ uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
sqlite_export_service_internal::StepStatement(
db_handle_, insert_brewery_stmt_, "Failed to insert SQLite brewery row");
const sqlite3_int64 brewery_id =
sqlite_export_service_internal::LastInsertRowId(db_handle_);
sqlite_export_service_internal::ResetStatement(insert_brewery_stmt_);
return sqlite_export_service_internal::LastInsertRowId(db_handle_);
sqlite_export_service_internal::Bind(
insert_brewery_address_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index =
sqlite_export_service_internal::kBreweryAddressBreweryIdBindIndex,
.value = brewery_id,
.action = "Failed to bind SQLite brewery address brewery id"});
sqlite_export_service_internal::Bind(
insert_brewery_address_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index =
sqlite_export_service_internal::kBreweryAddressCityIdBindIndex,
.value = city_id,
.action = "Failed to bind SQLite brewery address city id"});
sqlite_export_service_internal::Bind(
insert_brewery_address_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::
kBreweryAddressPostalCodeBindIndex,
.value = brewery.address.postal_code,
.action = "Failed to bind SQLite brewery address postal code"});
sqlite_export_service_internal::StepStatement(
db_handle_, insert_brewery_address_stmt_,
"Failed to insert SQLite brewery address row");
sqlite_export_service_internal::ResetStatement(insert_brewery_address_stmt_);
return static_cast<uint64_t>(brewery_id);
}

39
tooling/pipeline/src/services/sqlite/process_user_record.cc
View File

@@ -13,14 +13,8 @@ uint64_t SqliteExportService::ProcessRecord(const UserRecord& user) {
throw std::runtime_error("SQLite export service is not initialized");
}
const sqlite3_int64 location_id = ResolveLocationId(user.location);
const sqlite3_int64 city_id = ResolveCityId(user.address.city);
sqlite_export_service_internal::Bind(
insert_user_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index = sqlite_export_service_internal::kUserLocationIdBindIndex,
.value = location_id,
.action = "Failed to bind SQLite user location id"});
sqlite_export_service_internal::Bind(
insert_user_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
@@ -73,7 +67,36 @@ uint64_t SqliteExportService::ProcessRecord(const UserRecord& user) {
sqlite_export_service_internal::StepStatement(
db_handle_, insert_user_stmt_, "Failed to insert SQLite user row");
const sqlite3_int64 user_id =
sqlite_export_service_internal::LastInsertRowId(db_handle_);
sqlite_export_service_internal::ResetStatement(insert_user_stmt_);
return sqlite_export_service_internal::LastInsertRowId(db_handle_);
sqlite_export_service_internal::Bind(
insert_user_address_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index = sqlite_export_service_internal::kUserAddressUserIdBindIndex,
.value = user_id,
.action = "Failed to bind SQLite user address user id"});
sqlite_export_service_internal::Bind(
insert_user_address_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index = sqlite_export_service_internal::kUserAddressCityIdBindIndex,
.value = city_id,
.action = "Failed to bind SQLite user address city id"});
sqlite_export_service_internal::Bind(
insert_user_address_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index =
sqlite_export_service_internal::kUserAddressPostalCodeBindIndex,
.value = user.address.postal_code,
.action = "Failed to bind SQLite user address postal code"});
sqlite_export_service_internal::StepStatement(
db_handle_, insert_user_address_stmt_,
"Failed to insert SQLite user address row");
sqlite_export_service_internal::ResetStatement(insert_user_address_stmt_);
return static_cast<uint64_t>(user_id);
}

23
tooling/pipeline/src/services/xeger/xeger.cc Normal file
View File

@@ -0,0 +1,23 @@
/**
* @file services/xeger/xeger.cc
* @brief Public entry point: parses a regex pattern into an AST and walks it
* to emit a random matching string ("xeger" -- "regex" backwards).
*
* See xeger_internal.h for the parser/AST/generator this delegates to.
*/
#include "services/xeger/xeger.h"
#include <random>
#include <string>
#include <string_view>
#include "services/xeger/xeger_internal.h"
std::string GenerateStringFromRegex(std::string_view pattern, std::mt19937& rng,
const int unbounded_repeat_cap) {
xeger_internal::Parser parser(pattern);
const xeger_internal::NodePtr ast = parser.Parse();
xeger_internal::XegerGenerator generator(rng, unbounded_repeat_cap);
return generator.Run(*ast);
}

110
tooling/pipeline/src/services/xeger/xeger_generator.cc Normal file
View File

@@ -0,0 +1,110 @@
/**
* @file services/xeger/xeger_generator.cc
* @brief Visitor that walks a regex AST and emits a random matching string.
*/
#include "services/xeger/xeger_internal.h"
#include <cassert>
#include <limits>
#include <random>
#include <stdexcept>
#include <string>
namespace xeger_internal {
XegerGenerator::XegerGenerator(std::mt19937& rng, const int unbounded_cap)
: rng_(rng), unbounded_cap_(unbounded_cap) {}
std::string XegerGenerator::Run(const Node& root) {
out_.clear();
root.Accept(*this);
return out_;
}
void XegerGenerator::Visit(const Literal& node) { out_ += node.ch; }
void XegerGenerator::Visit(const AnyChar&) {
static constexpr int kPrintableLow = ' '; // empty space - character 32
static constexpr int kPrintableHigh = '~'; // tilde - character 126
std::uniform_int_distribution<int> dist(kPrintableLow, kPrintableHigh);
// assert that the rand number is in the char range
bool inCharRange = dist.min() >= std::numeric_limits<char>::min() &&
dist.max() <= std::numeric_limits<char>::max();
assert(inCharRange && "Random number is out of char range");
out_ += static_cast<char>(dist(rng_));
}
void XegerGenerator::Visit(const CharClass& node) {
out_ += PickFromClass(node);
}
void XegerGenerator::Visit(const Concat& node) {
for (const NodePtr& part : node.parts) {
part->Accept(*this);
}
}
void XegerGenerator::Visit(const Alternation& node) {
std::uniform_int_distribution<size_t> dist(0, node.options.size() - 1);
node.options[dist(rng_)]->Accept(*this);
}
void XegerGenerator::Visit(const Repeat& node) {
const int lo = node.min;
const int hi = (node.max < 0) ? node.min + unbounded_cap_ : node.max;
std::uniform_int_distribution<int> dist(lo, hi);
const int count = dist(rng_);
for (int i = 0; i < count; ++i) {
node.child->Accept(*this);
}
}
bool XegerGenerator::InRanges(const CharClass& node, const char c) {
for (const auto& [lo, hi] : node.ranges) {
if (c >= lo && c <= hi) {
return true;
}
}
return false;
}
char XegerGenerator::PickFromClass(const CharClass& node) {
if (!node.negated) {
int total = 0;
for (const auto& [lo, hi] : node.ranges) {
total += hi - lo + 1;
}
std::uniform_int_distribution<int> dist(0, total - 1);
int k = dist(rng_);
for (const auto& [lo, hi] : node.ranges) {
const int width = hi - lo + 1;
if (k < width) {
return static_cast<char>(lo + k);
}
k -= width;
}
return node.ranges.back().second; // unreachable
}
// Negated class: draw random printable characters until one falls
// outside the forbidden set.
static constexpr int kPrintableLow = 32;
static constexpr int kPrintableHigh = 126;
static constexpr int kMaxAttempts = 1000;
std::uniform_int_distribution<int> dist(kPrintableLow, kPrintableHigh);
for (int attempt = 0; attempt < kMaxAttempts; ++attempt) {
const char c = static_cast<char>(dist(rng_));
if (!InRanges(node, c)) {
return c;
}
}
throw std::runtime_error(
"GenerateStringFromRegex: negated class rules out every printable "
"character");
}
} // namespace xeger_internal

307
tooling/pipeline/src/services/xeger/xeger_parser.cc Normal file
View File

@@ -0,0 +1,307 @@
/**
* @file services/xeger/xeger_parser.cc
* @brief Recursive-descent parser that turns a regex pattern into an AST.
*/
#include "services/xeger/xeger_internal.h"
#include <cctype>
#include <format>
#include <memory>
#include <stdexcept>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
namespace xeger_internal {
Parser::Parser(std::string_view pattern) : src_(StripAnchors(pattern)) {}
NodePtr Parser::Parse() {
NodePtr node = ParseAlternation();
if (!AtEnd()) {
throw std::runtime_error(
std::format("GenerateStringFromRegex: unexpected '{}' at position {}",
Peek(), pos_));
}
return node;
}
std::string Parser::StripAnchors(std::string_view pattern) {
if (!pattern.empty() && pattern.front() == kAnchorStart) {
pattern.remove_prefix(1);
}
const bool has_trailing_dollar =
!pattern.empty() && pattern.back() == kAnchorEnd &&
(pattern.size() < 2 || pattern[pattern.size() - 2] != kEscape);
if (has_trailing_dollar) {
pattern.remove_suffix(1);
}
return std::string(pattern);
}
bool Parser::AtEnd() const { return pos_ >= src_.size(); }
char Parser::Peek() const { return src_[pos_]; }
char Parser::Advance() { return src_[pos_++]; }
bool Parser::Match(const char c) {
if (!AtEnd() && Peek() == c) {
++pos_;
return true;
}
return false;
}
void Parser::Expect(const char c) {
if (!Match(c)) {
throw std::runtime_error(std::format(
"GenerateStringFromRegex: expected '{}' at position {}", c, pos_));
}
}
NodePtr Parser::ParseAlternation() {
std::vector<NodePtr> options;
options.push_back(ParseConcat());
while (Match(kAlternation)) {
options.push_back(ParseConcat());
}
if (options.size() == 1) {
return std::move(options.front());
}
auto alternation = std::make_unique<Alternation>();
alternation->options = std::move(options);
return alternation;
}
NodePtr Parser::ParseConcat() {
auto concat = std::make_unique<Concat>();
while (!AtEnd() && Peek() != kAlternation && Peek() != kGroupClose) {
concat->parts.push_back(ParseRepeat());
}
if (concat->parts.size() == 1) {
return std::move(concat->parts.front());
}
return concat;
}
NodePtr Parser::ParseRepeat() {
NodePtr atom = ParseAtom();
if (AtEnd()) {
return atom;
}
int lo = 0;
int hi = -1;
switch (Peek()) {
case kStar:
Advance();
lo = 0;
hi = -1;
break;
case kPlus:
Advance();
lo = 1;
hi = -1;
break;
case kOptional:
Advance();
lo = 0;
hi = 1;
break;
case kBraceOpen:
return ParseBrace(std::move(atom));
default:
return atom;
}
auto repeat = std::make_unique<Repeat>();
repeat->child = std::move(atom);
repeat->min = lo;
repeat->max = hi;
return repeat;
}
NodePtr Parser::ParseBrace(NodePtr atom) {
Expect(kBraceOpen);
std::string lower_digits;
std::string upper_digits;
bool has_comma = false;
while (!AtEnd() && std::isdigit(static_cast<unsigned char>(Peek())) != 0) {
lower_digits += Advance();
}
if (Match(kBraceComma)) {
has_comma = true;
while (!AtEnd() &&
std::isdigit(static_cast<unsigned char>(Peek())) != 0) {
upper_digits += Advance();
}
}
Expect(kBraceClose);
auto repeat = std::make_unique<Repeat>();
repeat->child = std::move(atom);
repeat->min = lower_digits.empty() ? 0 : std::stoi(lower_digits);
if (!has_comma) {
repeat->max = repeat->min;
} else if (upper_digits.empty()) {
repeat->max = -1;
} else {
repeat->max = std::stoi(upper_digits);
}
if (repeat->max != -1 && repeat->max < repeat->min) {
throw std::runtime_error(
"GenerateStringFromRegex: repetition max is smaller than min");
}
return repeat;
}
NodePtr Parser::ParseAtom() {
if (AtEnd()) {
throw std::runtime_error(
"GenerateStringFromRegex: pattern ended unexpectedly");
}
const char c = Peek();
switch (c) {
case kGroupOpen: {
Advance();
// Non-capturing group "(?:...)" -- capture semantics don't apply
// here (nothing back-references a group), so it's treated exactly
// like a plain group once the "?:" marker is consumed.
if (!AtEnd() && Peek() == kNonCapturingMarker) {
if (pos_ + 1 < src_.size() && src_[pos_ + 1] == kNonCapturingColon) {
pos_ += 2;
} else {
throw std::runtime_error(std::format(
"GenerateStringFromRegex: unsupported group syntax at "
"position {} (only plain '(...)' and non-capturing "
"'(?:...)' groups are supported)",
pos_));
}
}
NodePtr inner = ParseAlternation();
Expect(kGroupClose);
return inner;
}
case kClassOpen:
return ParseCharClass();
case kAnyChar:
Advance();
return std::make_unique<AnyChar>();
case kEscape:
Advance();
return ParseEscape();
case kStar:
case kPlus:
case kOptional:
case kBraceOpen:
throw std::runtime_error(std::format(
"GenerateStringFromRegex: quantifier with no preceding item at "
"position {}",
pos_));
default:
Advance();
return std::make_unique<Literal>(c);
}
}
NodePtr Parser::MakeClass(std::vector<std::pair<char, char>> ranges,
const bool negated) {
auto char_class = std::make_unique<CharClass>();
char_class->ranges = std::move(ranges);
char_class->negated = negated;
return char_class;
}
NodePtr Parser::ParseEscape() {
if (AtEnd()) {
throw std::runtime_error(
"GenerateStringFromRegex: a backslash at the very end has nothing "
"to escape");
}
const char c = Advance();
switch (c) {
case 'd':
return MakeClass({{'0', '9'}}, false);
case 'D':
return MakeClass({{'0', '9'}}, true);
case 'w':
return MakeClass({{'a', 'z'}, {'A', 'Z'}, {'0', '9'}, {'_', '_'}},
false);
case 'W':
return MakeClass({{'a', 'z'}, {'A', 'Z'}, {'0', '9'}, {'_', '_'}},
true);
case 's':
return MakeClass({{' ', ' '}, {'\t', '\t'}, {'\n', '\n'}, {'\r', '\r'}},
false);
case 'S':
return MakeClass({{' ', ' '}, {'\t', '\t'}, {'\n', '\n'}, {'\r', '\r'}},
true);
case 'n':
return std::make_unique<Literal>('\n');
case 't':
return std::make_unique<Literal>('\t');
case 'r':
return std::make_unique<Literal>('\r');
default:
// \. \* \( \\ etc. -- the literal character itself.
return std::make_unique<Literal>(c);
}
}
NodePtr Parser::ParseCharClass() {
Expect(kClassOpen);
auto char_class = std::make_unique<CharClass>();
if (Match(kClassNegate)) {
char_class->negated = true;
}
if (AtEnd()) {
throw std::runtime_error(
"GenerateStringFromRegex: character class '[' is never closed");
}
while (!AtEnd() && Peek() != kClassClose) {
char lo = ClassChar();
const bool is_range = Peek() == kClassRange && pos_ + 1 < src_.size() &&
src_[pos_ + 1] != kClassClose;
if (is_range) {
Advance();
char hi = ClassChar();
if (hi < lo) {
std::swap(lo, hi);
}
char_class->ranges.emplace_back(lo, hi);
} else {
char_class->ranges.emplace_back(lo, lo);
}
}
Expect(kClassClose);
if (char_class->ranges.empty()) {
throw std::runtime_error(
"GenerateStringFromRegex: empty character class []");
}
return char_class;
}
char Parser::ClassChar() {
const char c = Advance();
if (c != kEscape) {
return c;
}
if (AtEnd()) {
throw std::runtime_error(
"GenerateStringFromRegex: backslash at end of character class");
}
const char escaped = Advance();
switch (escaped) {
case 'n':
return '\n';
case 't':
return '\t';
case 'r':
return '\r';
default:
return escaped;
}
}
} // namespace xeger_internal

9
web/backend/Database/Database.Seed/Database.Seed.csproj
View File

@@ -10,8 +10,15 @@
<ItemGroup>
<PackageReference Include="idunno.Password.Generator" Version="1.0.1" />
<PackageReference Include="Konscious.Security.Cryptography.Argon2" Version="1.3.1" />
<PackageReference Include="Microsoft.Data.SqlClient" Version="6.1.4" />
<PackageReference Include="Microsoft.Data.SqlClient" Version="7.0.2" />
<PackageReference Include="Microsoft.Data.Sqlite" Version="10.0.9" />
<PackageReference Include="dbup" Version="5.0.41" />
<!--
remove this override once microsoft ships official fix to vulnerability
https://github.com/advisories/GHSA-2m69-gcr7-jv3q
-->
<PackageReference Include="SQLitePCLRaw.lib.e_sqlite3" Version="3.50.3" />
</ItemGroup>
<ItemGroup>

54
web/backend/Database/Database.Seed/DatabaseHelpers.cs Normal file
View File

@@ -0,0 +1,54 @@
using Microsoft.Data.SqlClient;
namespace Database.Seed;
public class ConnectionStrings
{
/// <summary>
/// Builds a SQL Server connection string from the <c>DB_SERVER</c>, <c>DB_NAME</c>,
/// <c>DB_USER</c>, <c>DB_PASSWORD</c>, and <c>DB_TRUST_SERVER_CERTIFICATE</c>
/// environment variables.
/// </summary>
/// <returns>A fully built SQL Server connection string.</returns>
/// <exception cref="InvalidOperationException">
/// Thrown when <c>DB_SERVER</c>, <c>DB_NAME</c>, <c>DB_USER</c>, or <c>DB_PASSWORD</c>
/// is not set.
/// </exception>
private static string GetSqlServerConnectionString()
{
string server =
Environment.GetEnvironmentVariable("DB_SERVER")
?? throw new InvalidOperationException("DB_SERVER environment variable is not set");
string dbName =
Environment.GetEnvironmentVariable("DB_NAME")
?? throw new InvalidOperationException("DB_NAME environment variable is not set");
string user =
Environment.GetEnvironmentVariable("DB_USER")
?? throw new InvalidOperationException("DB_USER environment variable is not set");
string password =
Environment.GetEnvironmentVariable("DB_PASSWORD")
?? throw new InvalidOperationException("DB_PASSWORD environment variable is not set");
string trustServerCertificate =
Environment.GetEnvironmentVariable("DB_TRUST_SERVER_CERTIFICATE") ?? "True";
SqlConnectionStringBuilder builder = new()
{
DataSource = server,
InitialCatalog = dbName,
UserID = user,
Password = password,
TrustServerCertificate = bool.Parse(trustServerCertificate),
Encrypt = true,
};
return builder.ConnectionString;
}
public static string SqlServerConnectionString => GetSqlServerConnectionString();
public static string SqliteConnectionString => "Data Source=seed.sqlite";
}

19
web/backend/Database/Database.Seed/ISeeder.cs
View File

@@ -1,19 +0,0 @@
using Microsoft.Data.SqlClient;
namespace Database.Seed;
/// <summary>
/// Defines a unit of seed data that can be applied to the database. Implementations
/// should be safe to run against an already-seeded database (e.g. by checking for
/// existing data before inserting) and may depend on data created by seeders that run
/// before them.
/// </summary>
internal interface ISeeder
{
/// <summary>
/// Inserts this seeder's data into the database using the supplied open connection.
/// </summary>
/// <param name="connection">An open connection to the target database.</param>
/// <returns>A task that completes when seeding is finished.</returns>
Task SeedAsync(SqlConnection connection);
}

341
web/backend/Database/Database.Seed/LocationSeeder.cs
View File

@@ -1,341 +0,0 @@
using System.Data;
using Microsoft.Data.SqlClient;
namespace Database.Seed;
/// <summary>
/// Seeds the location hierarchy (countries, states/provinces, and cities) used to
/// associate other entities (e.g. breweries, users) with a geographic location.
/// Countries must be seeded before states/provinces, which must be seeded before
/// cities, since each level references its parent by code. The underlying stored
/// procedures (<c>USP_CreateCountry</c>, <c>USP_CreateStateProvince</c>,
/// <c>USP_CreateCity</c>) are expected to be idempotent, so re-running this seeder
/// against an already-seeded database is safe.
/// </summary>
internal class LocationSeeder : ISeeder
{
/// <summary>The set of countries to seed, identified by name and ISO 3166-1 code.</summary>
private static readonly IReadOnlyList<(string CountryName, string CountryCode)> Countries =
[
("Canada", "CA"),
("Mexico", "MX"),
("United States", "US"),
];
/// <summary>
/// The set of states/provinces to seed, each identified by name, ISO 3166-2 code,
/// and the ISO 3166-1 code of its parent country.
/// </summary>
private static IReadOnlyList<(
string StateProvinceName,
string StateProvinceCode,
string CountryCode
)> States { get; } =
[
("Alabama", "US-AL", "US"),
("Alaska", "US-AK", "US"),
("Arizona", "US-AZ", "US"),
("Arkansas", "US-AR", "US"),
("California", "US-CA", "US"),
("Colorado", "US-CO", "US"),
("Connecticut", "US-CT", "US"),
("Delaware", "US-DE", "US"),
("Florida", "US-FL", "US"),
("Georgia", "US-GA", "US"),
("Hawaii", "US-HI", "US"),
("Idaho", "US-ID", "US"),
("Illinois", "US-IL", "US"),
("Indiana", "US-IN", "US"),
("Iowa", "US-IA", "US"),
("Kansas", "US-KS", "US"),
("Kentucky", "US-KY", "US"),
("Louisiana", "US-LA", "US"),
("Maine", "US-ME", "US"),
("Maryland", "US-MD", "US"),
("Massachusetts", "US-MA", "US"),
("Michigan", "US-MI", "US"),
("Minnesota", "US-MN", "US"),
("Mississippi", "US-MS", "US"),
("Missouri", "US-MO", "US"),
("Montana", "US-MT", "US"),
("Nebraska", "US-NE", "US"),
("Nevada", "US-NV", "US"),
("New Hampshire", "US-NH", "US"),
("New Jersey", "US-NJ", "US"),
("New Mexico", "US-NM", "US"),
("New York", "US-NY", "US"),
("North Carolina", "US-NC", "US"),
("North Dakota", "US-ND", "US"),
("Ohio", "US-OH", "US"),
("Oklahoma", "US-OK", "US"),
("Oregon", "US-OR", "US"),
("Pennsylvania", "US-PA", "US"),
("Rhode Island", "US-RI", "US"),
("South Carolina", "US-SC", "US"),
("South Dakota", "US-SD", "US"),
("Tennessee", "US-TN", "US"),
("Texas", "US-TX", "US"),
("Utah", "US-UT", "US"),
("Vermont", "US-VT", "US"),
("Virginia", "US-VA", "US"),
("Washington", "US-WA", "US"),
("West Virginia", "US-WV", "US"),
("Wisconsin", "US-WI", "US"),
("Wyoming", "US-WY", "US"),
("District of Columbia", "US-DC", "US"),
("Puerto Rico", "US-PR", "US"),
("U.S. Virgin Islands", "US-VI", "US"),
("Guam", "US-GU", "US"),
("Northern Mariana Islands", "US-MP", "US"),
("American Samoa", "US-AS", "US"),
("Ontario", "CA-ON", "CA"),
("Québec", "CA-QC", "CA"),
("Nova Scotia", "CA-NS", "CA"),
("New Brunswick", "CA-NB", "CA"),
("Manitoba", "CA-MB", "CA"),
("British Columbia", "CA-BC", "CA"),
("Prince Edward Island", "CA-PE", "CA"),
("Saskatchewan", "CA-SK", "CA"),
("Alberta", "CA-AB", "CA"),
("Newfoundland and Labrador", "CA-NL", "CA"),
("Northwest Territories", "CA-NT", "CA"),
("Yukon", "CA-YT", "CA"),
("Nunavut", "CA-NU", "CA"),
("Aguascalientes", "MX-AGU", "MX"),
("Baja California", "MX-BCN", "MX"),
("Baja California Sur", "MX-BCS", "MX"),
("Campeche", "MX-CAM", "MX"),
("Chiapas", "MX-CHP", "MX"),
("Chihuahua", "MX-CHH", "MX"),
("Coahuila de Zaragoza", "MX-COA", "MX"),
("Colima", "MX-COL", "MX"),
("Durango", "MX-DUR", "MX"),
("Guanajuato", "MX-GUA", "MX"),
("Guerrero", "MX-GRO", "MX"),
("Hidalgo", "MX-HID", "MX"),
("Jalisco", "MX-JAL", "MX"),
("México State", "MX-MEX", "MX"),
("Michoacán de Ocampo", "MX-MIC", "MX"),
("Morelos", "MX-MOR", "MX"),
("Nayarit", "MX-NAY", "MX"),
("Nuevo León", "MX-NLE", "MX"),
("Oaxaca", "MX-OAX", "MX"),
("Puebla", "MX-PUE", "MX"),
("Querétaro", "MX-QUE", "MX"),
("Quintana Roo", "MX-ROO", "MX"),
("San Luis Potosí", "MX-SLP", "MX"),
("Sinaloa", "MX-SIN", "MX"),
("Sonora", "MX-SON", "MX"),
("Tabasco", "MX-TAB", "MX"),
("Tamaulipas", "MX-TAM", "MX"),
("Tlaxcala", "MX-TLA", "MX"),
("Veracruz de Ignacio de la Llave", "MX-VER", "MX"),
("Yucatán", "MX-YUC", "MX"),
("Zacatecas", "MX-ZAC", "MX"),
("Ciudad de México", "MX-CMX", "MX"),
];
/// <summary>
/// The set of cities to seed, each identified by name and the ISO 3166-2 code of its
/// parent state/province.
/// </summary>
private static IReadOnlyList<(string StateProvinceCode, string CityName)> Cities { get; } =
[
("US-CA", "Los Angeles"),
("US-CA", "San Diego"),
("US-CA", "San Francisco"),
("US-CA", "Sacramento"),
("US-TX", "Houston"),
("US-TX", "Dallas"),
("US-TX", "Austin"),
("US-TX", "San Antonio"),
("US-FL", "Miami"),
("US-FL", "Orlando"),
("US-FL", "Tampa"),
("US-NY", "New York"),
("US-NY", "Buffalo"),
("US-NY", "Rochester"),
("US-IL", "Chicago"),
("US-IL", "Springfield"),
("US-PA", "Philadelphia"),
("US-PA", "Pittsburgh"),
("US-AZ", "Phoenix"),
("US-AZ", "Tucson"),
("US-CO", "Denver"),
("US-CO", "Colorado Springs"),
("US-MA", "Boston"),
("US-MA", "Worcester"),
("US-WA", "Seattle"),
("US-WA", "Spokane"),
("US-GA", "Atlanta"),
("US-GA", "Savannah"),
("US-NV", "Las Vegas"),
("US-NV", "Reno"),
("US-MI", "Detroit"),
("US-MI", "Grand Rapids"),
("US-MN", "Minneapolis"),
("US-MN", "Saint Paul"),
("US-OH", "Columbus"),
("US-OH", "Cleveland"),
("US-OR", "Portland"),
("US-OR", "Salem"),
("US-TN", "Nashville"),
("US-TN", "Memphis"),
("US-VA", "Richmond"),
("US-VA", "Virginia Beach"),
("US-MD", "Baltimore"),
("US-MD", "Frederick"),
("US-DC", "Washington"),
("US-UT", "Salt Lake City"),
("US-UT", "Provo"),
("US-LA", "New Orleans"),
("US-LA", "Baton Rouge"),
("US-KY", "Louisville"),
("US-KY", "Lexington"),
("US-IA", "Des Moines"),
("US-IA", "Cedar Rapids"),
("US-OK", "Oklahoma City"),
("US-OK", "Tulsa"),
("US-NE", "Omaha"),
("US-NE", "Lincoln"),
("US-MO", "Kansas City"),
("US-MO", "St. Louis"),
("US-NC", "Charlotte"),
("US-NC", "Raleigh"),
("US-SC", "Columbia"),
("US-SC", "Charleston"),
("US-WI", "Milwaukee"),
("US-WI", "Madison"),
("US-MN", "Duluth"),
("US-AK", "Anchorage"),
("US-HI", "Honolulu"),
("CA-ON", "Toronto"),
("CA-ON", "Ottawa"),
("CA-QC", "Montréal"),
("CA-QC", "Québec City"),
("CA-BC", "Vancouver"),
("CA-BC", "Victoria"),
("CA-AB", "Calgary"),
("CA-AB", "Edmonton"),
("CA-MB", "Winnipeg"),
("CA-NS", "Halifax"),
("CA-SK", "Saskatoon"),
("CA-SK", "Regina"),
("CA-NB", "Moncton"),
("CA-NB", "Saint John"),
("CA-PE", "Charlottetown"),
("CA-NL", "St. John's"),
("CA-ON", "Hamilton"),
("CA-ON", "London"),
("CA-QC", "Gatineau"),
("CA-QC", "Laval"),
("CA-BC", "Kelowna"),
("CA-AB", "Red Deer"),
("CA-MB", "Brandon"),
("MX-CMX", "Ciudad de México"),
("MX-JAL", "Guadalajara"),
("MX-NLE", "Monterrey"),
("MX-PUE", "Puebla"),
("MX-ROO", "Cancún"),
("MX-GUA", "Guanajuato"),
("MX-MIC", "Morelia"),
("MX-BCN", "Tijuana"),
("MX-JAL", "Zapopan"),
("MX-NLE", "San Nicolás"),
("MX-CAM", "Campeche"),
("MX-TAB", "Villahermosa"),
("MX-VER", "Veracruz"),
("MX-OAX", "Oaxaca"),
("MX-SLP", "San Luis Potosí"),
("MX-CHH", "Chihuahua"),
("MX-AGU", "Aguascalientes"),
("MX-MEX", "Toluca"),
("MX-COA", "Saltillo"),
("MX-BCS", "La Paz"),
("MX-NAY", "Tepic"),
("MX-ZAC", "Zacatecas"),
];
/// <summary>
/// Seeds all countries, then states/provinces, then cities, in that order, so that
/// each level's parent reference already exists by the time it is created.
/// </summary>
/// <param name="connection">An open connection to the target database.</param>
/// <returns>A task that completes when all locations have been seeded.</returns>
public async Task SeedAsync(SqlConnection connection)
{
foreach ((string countryName, string countryCode) in Countries)
await CreateCountryAsync(connection, countryName, countryCode);
foreach ((string stateProvinceName, string stateProvinceCode, string countryCode) in States)
await CreateStateProvinceAsync(
connection,
stateProvinceName,
stateProvinceCode,
countryCode
);
foreach ((string stateProvinceCode, string cityName) in Cities)
await CreateCityAsync(connection, cityName, stateProvinceCode);
}
/// <summary>Creates a single country by invoking the <c>dbo.USP_CreateCountry</c> stored procedure.</summary>
/// <param name="connection">An open connection to the target database.</param>
/// <param name="countryName">The display name of the country.</param>
/// <param name="countryCode">The ISO 3166-1 code of the country.</param>
/// <returns>A task that completes when the country has been created.</returns>
private static async Task CreateCountryAsync(
SqlConnection connection,
string countryName,
string countryCode
)
{
await using SqlCommand command = new("dbo.USP_CreateCountry", connection);
command.CommandType = CommandType.StoredProcedure;
command.Parameters.AddWithValue("@CountryName", countryName);
command.Parameters.AddWithValue("@ISO3166_1", countryCode);
await command.ExecuteNonQueryAsync();
}
/// <summary>Creates a single state/province by invoking the <c>dbo.USP_CreateStateProvince</c> stored procedure.</summary>
/// <param name="connection">An open connection to the target database.</param>
/// <param name="stateProvinceName">The display name of the state/province.</param>
/// <param name="stateProvinceCode">The ISO 3166-2 code of the state/province.</param>
/// <param name="countryCode">The ISO 3166-1 code of the parent country, which must already exist.</param>
/// <returns>A task that completes when the state/province has been created.</returns>
private static async Task CreateStateProvinceAsync(
SqlConnection connection,
string stateProvinceName,
string stateProvinceCode,
string countryCode
)
{
await using SqlCommand command = new("dbo.USP_CreateStateProvince", connection);
command.CommandType = CommandType.StoredProcedure;
command.Parameters.AddWithValue("@StateProvinceName", stateProvinceName);
command.Parameters.AddWithValue("@ISO3166_2", stateProvinceCode);
command.Parameters.AddWithValue("@CountryCode", countryCode);
await command.ExecuteNonQueryAsync();
}
/// <summary>Creates a single city by invoking the <c>dbo.USP_CreateCity</c> stored procedure.</summary>
/// <param name="connection">An open connection to the target database.</param>
/// <param name="cityName">The display name of the city.</param>
/// <param name="stateProvinceCode">The ISO 3166-2 code of the parent state/province, which must already exist.</param>
/// <returns>A task that completes when the city has been created.</returns>
private static async Task CreateCityAsync(
SqlConnection connection,
string cityName,
string stateProvinceCode
)
{
await using SqlCommand command = new("dbo.USP_CreateCity", connection);
command.CommandType = CommandType.StoredProcedure;
command.Parameters.AddWithValue("@CityName", cityName);
command.Parameters.AddWithValue("@StateProvinceCode", stateProvinceCode);
await command.ExecuteNonQueryAsync();
}
}

106
web/backend/Database/Database.Seed/Program.cs
View File

@@ -1,106 +0,0 @@
using Database.Seed;
using Microsoft.Data.SqlClient;
// Entry point for the database seeding utility. Connects to the target database
// (retrying on transient failures), then runs each registered ISeeder in order to
// populate location and user data.
/// <summary>
/// Builds a SQL Server connection string from the <c>DB_SERVER</c>, <c>DB_NAME</c>,
/// <c>DB_USER</c>, <c>DB_PASSWORD</c>, and <c>DB_TRUST_SERVER_CERTIFICATE</c>
/// environment variables.
/// </summary>
/// <returns>A fully built SQL Server connection string.</returns>
/// <exception cref="InvalidOperationException">
/// Thrown when <c>DB_SERVER</c>, <c>DB_NAME</c>, <c>DB_USER</c>, or <c>DB_PASSWORD</c>
/// is not set.
/// </exception>
string BuildConnectionString()
{
string server =
Environment.GetEnvironmentVariable("DB_SERVER")
?? throw new InvalidOperationException("DB_SERVER environment variable is not set");
string dbName =
Environment.GetEnvironmentVariable("DB_NAME")
?? throw new InvalidOperationException("DB_NAME environment variable is not set");
string user =
Environment.GetEnvironmentVariable("DB_USER")
?? throw new InvalidOperationException("DB_USER environment variable is not set");
string password =
Environment.GetEnvironmentVariable("DB_PASSWORD")
?? throw new InvalidOperationException("DB_PASSWORD environment variable is not set");
string trustServerCertificate =
Environment.GetEnvironmentVariable("DB_TRUST_SERVER_CERTIFICATE") ?? "True";
SqlConnectionStringBuilder builder = new()
{
DataSource = server,
InitialCatalog = dbName,
UserID = user,
Password = password,
TrustServerCertificate = bool.Parse(trustServerCertificate),
Encrypt = true,
};
return builder.ConnectionString;
}
try
{
string connectionString = BuildConnectionString();
Console.WriteLine("Attempting to connect to database...");
// Retry logic for database connection
SqlConnection? connection = null;
int maxRetries = 10;
int retryDelayMs = 2000;
for (int attempt = 1; attempt <= maxRetries; attempt++)
try
{
connection = new SqlConnection(connectionString);
await connection.OpenAsync();
Console.WriteLine($"Connected to database successfully on attempt {attempt}.");
break;
}
catch (SqlException ex) when (attempt < maxRetries)
{
Console.WriteLine($"Connection attempt {attempt}/{maxRetries} failed: {ex.Message}");
Console.WriteLine($"Retrying in {retryDelayMs}ms...");
await Task.Delay(retryDelayMs);
connection?.Dispose();
connection = null;
}
if (connection == null)
throw new Exception($"Failed to connect to database after {maxRetries} attempts.");
Console.WriteLine("Starting seeding...");
using (connection)
{
ISeeder[] seeders = [new LocationSeeder(), new UserSeeder()];
foreach (ISeeder seeder in seeders)
{
Console.WriteLine($"Seeding {seeder.GetType().Name}...");
await seeder.SeedAsync(connection);
Console.WriteLine($"{seeder.GetType().Name} seeded.");
}
Console.WriteLine("Seed completed successfully.");
}
return 0;
}
catch (Exception ex)
{
Console.Error.WriteLine("Seed failed:");
Console.Error.WriteLine(ex);
return 1;
}

303
web/backend/Database/Database.Seed/UserSeeder.cs
View File

@@ -1,303 +0,0 @@
using System.Data;
using System.Security.Cryptography;
using System.Text;
using idunno.Password;
using Konscious.Security.Cryptography;
using Microsoft.Data.SqlClient;
namespace Database.Seed;
/// <summary>
/// Seeds user accounts, credentials, and verification records. Creates one fixed
/// "Test User" account (<c>test.user@thebiergarten.app</c> / password <c>"password"</c>)
/// for testing, followed by a randomly-generated account for each name in
/// <see cref="SeedNames" />, each with a random password and date of birth. Skips
/// adding a verification record for a user that already has one, so this seeder is
/// safe to re-run, though re-running will still attempt to register (and may fail on)
/// users that already exist.
/// </summary>
internal class UserSeeder : ISeeder
{
/// <summary>The first/last name pairs used to generate seed user accounts.</summary>
private static readonly IReadOnlyList<(string FirstName, string LastName)> SeedNames =
[
("Aarya", "Mathews"),
("Aiden", "Wells"),
("Aleena", "Gonzalez"),
("Alessandra", "Nelson"),
("Amari", "Tucker"),
("Ameer", "Huff"),
("Amirah", "Hicks"),
("Analia", "Dominguez"),
("Anne", "Jenkins"),
("Apollo", "Davis"),
("Arianna", "White"),
("Aubree", "Moore"),
("Aubrielle", "Raymond"),
("Aydin", "Odom"),
("Bowen", "Casey"),
("Brock", "Huber"),
("Caiden", "Strong"),
("Cecilia", "Rosales"),
("Celeste", "Barber"),
("Chance", "Small"),
("Clara", "Roberts"),
("Collins", "Brandt"),
("Damir", "Wallace"),
("Declan", "Crawford"),
("Dennis", "Decker"),
("Dylan", "Lang"),
("Eliza", "Kane"),
("Elle", "Poole"),
("Elliott", "Miles"),
("Emelia", "Lucas"),
("Emilia", "Simpson"),
("Emmett", "Lugo"),
("Ethan", "Stephens"),
("Etta", "Woods"),
("Gael", "Moran"),
("Grant", "Benson"),
("Gwen", "James"),
("Huxley", "Chen"),
("Isabella", "Fisher"),
("Ivan", "Mathis"),
("Jamir", "McMillan"),
("Jaxson", "Shields"),
("Jimmy", "Richmond"),
("Josiah", "Flores"),
("Kaden", "Enriquez"),
("Kai", "Lawson"),
("Karsyn", "Adkins"),
("Karsyn", "Proctor"),
("Kayden", "Henson"),
("Kaylie", "Spears"),
("Kinslee", "Jones"),
("Kora", "Guerra"),
("Lane", "Skinner"),
("Laylani", "Christian"),
("Ledger", "Carroll"),
("Leilany", "Small"),
("Leland", "McCall"),
("Leonard", "Calhoun"),
("Levi", "Ochoa"),
("Lillie", "Vang"),
("Lola", "Sheppard"),
("Luciana", "Poole"),
("Maddox", "Hughes"),
("Mara", "Blackwell"),
("Marcellus", "Bartlett"),
("Margo", "Koch"),
("Maurice", "Gibson"),
("Maxton", "Dodson"),
("Mia", "Parrish"),
("Millie", "Fuentes"),
("Nellie", "Villanueva"),
("Nicolas", "Mata"),
("Nicolas", "Miller"),
("Oakleigh", "Foster"),
("Octavia", "Pierce"),
("Paisley", "Allison"),
("Quincy", "Andersen"),
("Quincy", "Frazier"),
("Raiden", "Roberts"),
("Raquel", "Lara"),
("Rudy", "McIntosh"),
("Salvador", "Stein"),
("Samantha", "Dickson"),
("Solomon", "Richards"),
("Sylvia", "Hanna"),
("Talia", "Trujillo"),
("Thalia", "Farrell"),
("Trent", "Mayo"),
("Trinity", "Cummings"),
("Ty", "Perry"),
("Tyler", "Romero"),
("Valeria", "Pierce"),
("Vance", "Neal"),
("Whitney", "Bell"),
("Wilder", "Graves"),
("William", "Logan"),
("Zara", "Wilkinson"),
("Zaria", "Gibson"),
("Zion", "Watkins"),
("Zoie", "Armstrong"),
];
/// <summary>
/// Registers a fixed test user account followed by one randomly-generated account
/// per entry in <see cref="SeedNames" />, adding a user verification record for each
/// newly created account that does not already have one. Progress counts are
/// written to the console on completion.
/// </summary>
/// <param name="connection">An open connection to the target database.</param>
/// <returns>A task that completes when all users have been seeded.</returns>
public async Task SeedAsync(SqlConnection connection)
{
PasswordGenerator generator = new();
Random rng = new();
int createdUsers = 0;
int createdCredentials = 0;
int createdVerifications = 0;
// create a known user for testing purposes
{
const string firstName = "Test";
const string lastName = "User";
const string email = "test.user@thebiergarten.app";
DateTime dob = new(1985, 03, 01);
string hash = GeneratePasswordHash("password");
await RegisterUserAsync(
connection,
$"{firstName}.{lastName}",
firstName,
lastName,
dob,
email,
hash
);
}
foreach ((string firstName, string lastName) in SeedNames)
{
// prepare user fields
string username = $"{firstName[0]}.{lastName}";
string email = $"{firstName}.{lastName}@thebiergarten.app";
DateTime dob = GenerateDateOfBirth(rng);
// generate a password and hash it
string pwd = generator.Generate(64, 10, 10);
string hash = GeneratePasswordHash(pwd);
// register the user (creates account + credential)
Guid id = await RegisterUserAsync(
connection,
username,
firstName,
lastName,
dob,
email,
hash
);
createdUsers++;
createdCredentials++;
// add user verification
if (await HasUserVerificationAsync(connection, id))
continue;
await AddUserVerificationAsync(connection, id);
createdVerifications++;
}
Console.WriteLine($"Created {createdUsers} user accounts.");
Console.WriteLine($"Added {createdCredentials} user credentials.");
Console.WriteLine($"Added {createdVerifications} user verifications.");
}
/// <summary>
/// Registers a new user account and its credential by invoking the
/// <c>dbo.USP_RegisterUser</c> stored procedure.
/// </summary>
/// <param name="connection">An open connection to the target database.</param>
/// <param name="username">The unique username for the account.</param>
/// <param name="firstName">The user's first name.</param>
/// <param name="lastName">The user's last name.</param>
/// <param name="dateOfBirth">The user's date of birth.</param>
/// <param name="email">The user's email address.</param>
/// <param name="hash">The salted password hash, as produced by <see cref="GeneratePasswordHash" />.</param>
/// <returns>The newly created user account's <see cref="Guid" /> identifier.</returns>
private static async Task<Guid> RegisterUserAsync(
SqlConnection connection,
string username,
string firstName,
string lastName,
DateTime dateOfBirth,
string email,
string hash
)
{
await using SqlCommand command = new("dbo.USP_RegisterUser", connection);
command.CommandType = CommandType.StoredProcedure;
command.Parameters.Add("@Username", SqlDbType.VarChar, 64).Value = username;
command.Parameters.Add("@FirstName", SqlDbType.NVarChar, 128).Value = firstName;
command.Parameters.Add("@LastName", SqlDbType.NVarChar, 128).Value = lastName;
command.Parameters.Add("@DateOfBirth", SqlDbType.DateTime).Value = dateOfBirth;
command.Parameters.Add("@Email", SqlDbType.VarChar, 128).Value = email;
command.Parameters.Add("@Hash", SqlDbType.NVarChar, -1).Value = hash;
object? result = await command.ExecuteScalarAsync();
return (Guid)result!;
}
/// <summary>
/// Hashes a plaintext password using Argon2id with a randomly generated 16-byte
/// salt, a degree of parallelism matching the available processor count, 64 MB of
/// memory, and 4 iterations.
/// </summary>
/// <param name="pwd">The plaintext password to hash.</param>
/// <returns>A string containing the base64-encoded salt and hash, separated by a colon.</returns>
private static string GeneratePasswordHash(string pwd)
{
byte[] salt = RandomNumberGenerator.GetBytes(16);
Argon2id argon2 = new(Encoding.UTF8.GetBytes(pwd))
{
Salt = salt,
DegreeOfParallelism = Math.Max(Environment.ProcessorCount, 1),
MemorySize = 65536,
Iterations = 4,
};
byte[] hash = argon2.GetBytes(32);
return $"{Convert.ToBase64String(salt)}:{Convert.ToBase64String(hash)}";
}
/// <summary>Checks whether a user verification record already exists for the given user account.</summary>
/// <param name="connection">An open connection to the target database.</param>
/// <param name="userAccountId">The user account identifier to check.</param>
/// <returns><c>true</c> if a verification record already exists; otherwise <c>false</c>.</returns>
private static async Task<bool> HasUserVerificationAsync(
SqlConnection connection,
Guid userAccountId
)
{
const string sql = """
SELECT 1
FROM dbo.UserVerification
WHERE UserAccountId = @UserAccountId;
""";
await using SqlCommand command = new(sql, connection);
command.Parameters.AddWithValue("@UserAccountId", userAccountId);
object? result = await command.ExecuteScalarAsync();
return result is not null;
}
/// <summary>Creates a user verification record by invoking the <c>dbo.USP_CreateUserVerification</c> stored procedure.</summary>
/// <param name="connection">An open connection to the target database.</param>
/// <param name="userAccountId">The identifier of the user account to verify.</param>
/// <returns>A task that completes when the verification record has been created.</returns>
private static async Task AddUserVerificationAsync(SqlConnection connection, Guid userAccountId)
{
await using SqlCommand command = new("dbo.USP_CreateUserVerification", connection);
command.CommandType = CommandType.StoredProcedure;
command.Parameters.AddWithValue("@UserAccountID_", userAccountId);
await command.ExecuteNonQueryAsync();
}
/// <summary>
/// Generates a random date of birth corresponding to an age between 19 and 48
/// years (inclusive), with a random day offset within that birth year.
/// </summary>
/// <param name="random">The random number source to use.</param>
/// <returns>A randomly generated date of birth.</returns>
private static DateTime GenerateDateOfBirth(Random random)
{
int age = 19 + random.Next(0, 30);
DateTime baseDate = DateTime.UtcNow.Date.AddYears(-age);
int offsetDays = random.Next(0, 365);
return baseDate.AddDays(-offsetDays);
}
}

18
web/backend/Database/Program.cs Normal file
View File

@@ -0,0 +1,18 @@
namespace Database.Seed;
public interface IExportDatabase
{
void SeedLocations(IEnumerable<Location> locations);
void SeedUsers(IEnumerable<User> users);
void SeedBreweries(IEnumerable<Brewery> breweries);
}
public interface IImportDatabase
{
IEnumerable<Location> GetLocations();
IEnumerable<User> GetUsers();
IEnumerable<Brewery> GetBreweries();
}

2
web/backend/Infrastructure/Infrastructure.Jwt/JwtInfrastructure.cs
View File

@@ -88,7 +88,7 @@ public class JwtInfrastructure : ITokenInfrastructure
return new ClaimsPrincipal(result.ClaimsIdentity);
}
catch (Exception e)
catch (Exception)
{
throw new UnauthorizedException("Invalid token");
}

2
web/backend/Infrastructure/Infrastructure.Sql/Infrastructure.Sql.csproj
View File

@@ -6,7 +6,7 @@
<RootNamespace>Infrastructure.Sql</RootNamespace>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Data.SqlClient" Version="6.1.4" />
<PackageReference Include="Microsoft.Data.SqlClient" Version="7.0.2" />
<PackageReference Include="Microsoft.SqlServer.Types" Version="160.1000.6" />
<PackageReference Include="System.Data.SqlClient" Version="4.9.0" />
<PackageReference