AaronPo/the-biergarten-app
1
0
Fork 0
mirror of https://github.com/aaronpo97/the-biergarten-app.git synced 2026-10-15 18:14:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Pipeline: City rename, add address/postal-code generation, add xeger generation (#232)

Browse Source
This commit is contained in:
Aaron Po
2026-07-19 03:59:06 -04:00
committed by GitHub
parent fbcf438381
commit f020671703
56 changed files with 2454 additions and 1151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
tooling/pipeline/src/services/curated_data/curated_json_data_service.cc
View File

@@ -1,6 +1,6 @@
/**
* @file json_handling/json_loader.cc
* @brief Parses curated location JSON input into strongly typed Location
* @brief Parses curated location JSON input into strongly typed City
* records with strict field validation and descriptive error reporting.
*/
@@ -30,16 +30,6 @@ static std::string ReadRequiredString(const boost::json::object& object,
return std::string(text);
}
static double ReadRequiredNumber(const boost::json::object& object,
const char* key) {
const boost::json::value* value = object.if_contains(key);
if (value == nullptr || !value->is_number()) {
throw std::runtime_error(
std::format("Missing or invalid numeric field: {}", key));
}
return value->to_number<double>();
}
static std::vector<std::string> ReadRequiredStringArray(
const boost::json::object& object, const char* key) {
const boost::json::value* value = object.if_contains(key);
@@ -61,6 +51,21 @@ static std::vector<std::string> ReadRequiredStringArray(
return items;
}
static PostalCodeSpec ReadPostalCodeSpec(const boost::json::object& object) {
const boost::json::value* value = object.if_contains("postal_code");
if (value == nullptr || !value->is_object()) {
throw std::runtime_error("Missing or invalid object field: postal_code");
}
const auto& postal_code = value->as_object();
return PostalCodeSpec{
.country_format_regex =
ReadRequiredString(postal_code, "country_format_regex"),
.city_regexes = ReadRequiredStringArray(postal_code, "city_regex"),
.examples = ReadRequiredStringArray(postal_code, "examples"),
};
}
namespace {
boost::json::value ParseJsonFile(const std::filesystem::path& filepath,
@@ -133,15 +138,14 @@ const LocationsList& CuratedJsonDataService::LoadLocations() {
}
const auto& object = item.as_object();
locations.push_back(Location{
locations.push_back(City{
.city = ReadRequiredString(object, "city"),
.state_province = ReadRequiredString(object, "state_province"),
.iso3166_2 = ReadRequiredString(object, "iso3166_2"),
.country = ReadRequiredString(object, "country"),
.iso3166_1 = ReadRequiredString(object, "iso3166_1"),
.local_languages = ReadRequiredStringArray(object, "local_languages"),
.latitude = ReadRequiredNumber(object, "latitude"),
.longitude = ReadRequiredNumber(object, "longitude"),
.postal_code = ReadPostalCodeSpec(object),
});
}
cache_.locations = std::move(locations);

72
tooling/pipeline/src/services/curated_data/mock_curated_data_service.cc
View File

@@ -7,38 +7,46 @@
MockCuratedDataService::MockCuratedDataService()
: locations_{
Location{.city = "Portland",
.state_province = "Oregon",
.iso3166_2 = "US-OR",
.country = "United States",
.iso3166_1 = "US",
.local_languages = {"en"},
.latitude = 45.5152,
.longitude = -122.6784},
Location{.city = "Munich",
.state_province = "Bavaria",
.iso3166_2 = "DE-BY",
.country = "Germany",
.iso3166_1 = "DE",
.local_languages = {"de"},
.latitude = 48.1351,
.longitude = 11.5820},
Location{.city = "Lyon",
.state_province = "Auvergne-Rhone-Alpes",
.iso3166_2 = "FR-ARA",
.country = "France",
.iso3166_1 = "FR",
.local_languages = {"fr"},
.latitude = 45.7640,
.longitude = 4.8357},
Location{.city = "Brussels",
.state_province = "Brussels-Capital",
.iso3166_2 = "BE-BRU",
.country = "Belgium",
.iso3166_1 = "BE",
.local_languages = {"nl", "fr"},
.latitude = 50.8503,
.longitude = 4.3517},
City{.city = "Portland",
.state_province = "Oregon",
.iso3166_2 = "US-OR",
.country = "United States",
.iso3166_1 = "US",
.local_languages = {"en"},
.postal_code =
PostalCodeSpec{.country_format_regex = R"(^\d{5}$)",
.city_regexes = {R"(^972\d{2}$)"},
.examples = {"97201", "97202", "97209"}}},
City{.city = "Munich",
.state_province = "Bavaria",
.iso3166_2 = "DE-BY",
.country = "Germany",
.iso3166_1 = "DE",
.local_languages = {"de"},
.postal_code =
PostalCodeSpec{.country_format_regex = R"(^\d{5}$)",
.city_regexes = {R"(^8\d{4}$)"},
.examples = {"80331", "80333", "81667"}}},
City{.city = "Lyon",
.state_province = "Auvergne-Rhone-Alpes",
.iso3166_2 = "FR-ARA",
.country = "France",
.iso3166_1 = "FR",
.local_languages = {"fr"},
.postal_code =
PostalCodeSpec{.country_format_regex = R"(^\d{5}$)",
.city_regexes = {R"(^690\d{2}$)"},
.examples = {"69001", "69002", "69007"}}},
City{.city = "Brussels",
.state_province = "Brussels-Capital",
.iso3166_2 = "BE-BRU",
.country = "Belgium",
.iso3166_1 = "BE",
.local_languages = {"nl", "fr"},
.postal_code =
PostalCodeSpec{.country_format_regex = R"(^\d{4}$)",
.city_regexes = {R"(^1\d{3}$)"},
.examples = {"1000", "1050", "1180"}}},
},
personas_{
UserPersona{.name = "Hophead Explorer",

2
tooling/pipeline/src/services/enrichment/wikipedia/get_summary.cc
View File

@@ -11,7 +11,7 @@
#include "services/enrichment/wikipedia_service.h"
std::string WikipediaEnrichmentService::GetLocationContext(
const Location& loc) {
const City& loc) {
using namespace std::literals::chrono_literals;
if (!this->client_) {
if (logger_) {

44
tooling/pipeline/src/services/postal_code/xeger_postal_code_service.cc Normal file
View File

@@ -0,0 +1,44 @@
/**
* @file services/postal_code/xeger_postal_code_service.cc
* @brief XegerPostalCodeService::GeneratePostalCode() implementation.
*/
#include "services/postal_code/xeger_postal_code_service.h"
#include <format>
#include <random>
#include <stdexcept>
#include <string>
#include <string_view>
#include <vector>
#include "services/xeger/xeger.h"
XegerPostalCodeService::XegerPostalCodeService()
: rng_(std::random_device{}()) {}
std::string XegerPostalCodeService::GeneratePostalCode(const City& city) {
const PostalCodeSpec& spec = city.postal_code;
std::vector<std::string_view> city_patterns;
for (const std::string& pattern : spec.city_regexes) {
if (!pattern.empty()) {
city_patterns.push_back(pattern);
}
}
std::string_view pattern;
if (!city_patterns.empty()) {
std::uniform_int_distribution<size_t> dist(0, city_patterns.size() - 1);
pattern = city_patterns[dist(rng_)];
} else if (!spec.country_format_regex.empty()) {
pattern = spec.country_format_regex;
} else {
throw std::runtime_error(std::format(
"XegerPostalCodeService: no postal-code pattern available for "
"city '{}'",
city.city));
}
return GenerateStringFromRegex(pattern, rng_);
}

11
tooling/pipeline/src/services/prompt_directory.cc
View File

@@ -15,9 +15,7 @@
#include <string_view>
#include <utility>
// ---------------------------------------------------------------------------
// PromptDirectory
// ---------------------------------------------------------------------------
PromptDirectory::PromptDirectory(const std::filesystem::path& prompt_dir)
: PromptDirectory(prompt_dir, nullptr) {}
@@ -27,21 +25,21 @@ PromptDirectory::PromptDirectory(const std::filesystem::path& prompt_dir,
: prompt_dir_(prompt_dir), logger_(std::move(logger)) {
std::error_code ec;
// Scenario 4: directory must exist.
// Directory must exist.
if (!std::filesystem::exists(prompt_dir_, ec) || ec) {
throw std::runtime_error(
"PromptDirectory: prompt directory does not exist: " +
prompt_dir_.string());
}
// Scenario 4: path must be a directory, not a file.
// Path must be a directory, not a file.
if (!std::filesystem::is_directory(prompt_dir_, ec) || ec) {
throw std::runtime_error(
"PromptDirectory: prompt directory path is not a directory: " +
prompt_dir_.string());
}
// Scenario 4: directory must be readable (probe with directory_iterator).
// Directory must be readable (probe with directory_iterator).
std::filesystem::directory_iterator probe(prompt_dir_, ec);
if (ec) {
throw std::runtime_error(std::format(
@@ -68,7 +66,7 @@ std::string PromptDirectory::Load(std::string_view key) {
return cache_it->second;
}
// Scenario 3: resolve <prompt_dir>/<key>.md and require it to exist.
// Resolve <prompt_dir>/<key>.md and require it to exist.
const std::filesystem::path file_path =
prompt_dir_ / std::filesystem::path(std::format("{}.md", key_str));
@@ -98,6 +96,7 @@ std::string PromptDirectory::Load(std::string_view key) {
key_str, file_path.string(), content.size())});
}
// Cache the loaded content for future calls.
cache_.emplace(key_str, content);
return content;
}

6
tooling/pipeline/src/services/sqlite/finalize.cc
View File

@@ -14,9 +14,11 @@ void SqliteExportService::Finalize() {
}
try {
insert_user_address_stmt_.reset();
insert_user_stmt_.reset();
insert_brewery_address_stmt_.reset();
insert_brewery_stmt_.reset();
insert_location_stmt_.reset();
insert_city_stmt_.reset();
if (transaction_open_) {
sqlite_export_service_internal::ExecSql(
db_handle_, "COMMIT;", "Failed to commit SQLite transaction");
@@ -24,7 +26,7 @@ void SqliteExportService::Finalize() {
}
db_handle_.reset();
location_cache_.clear();
city_cache_.clear();
} catch (...) {
RollbackAndCloseNoThrow();
throw;

3
tooling/pipeline/src/services/sqlite/helpers/sqlite_statement_helpers.cc
View File

@@ -88,7 +88,8 @@ sqlite3_int64 LastInsertRowId(const SqliteDatabaseHandle& db_handle) {
}
std::string SerializeVector(const std::vector<std::string>& str_vec) {
boost::json::array array(str_vec.size());
boost::json::array array;
array.reserve(str_vec.size());
for (const auto& s : str_vec) {
array.emplace_back(s);
}

31
tooling/pipeline/src/services/sqlite/initialize.cc
View File

@@ -10,7 +10,6 @@
#include <string>
#include "services/database/sqlite_export_service.h"
#include "services/database/sqlite_export_service_helpers.h"
std::filesystem::path SqliteExportService::BuildDatabasePath() const {
std::filesystem::path base_filename("biergarten_seed_" + run_timestamp_utc_ +
@@ -28,26 +27,40 @@ std::filesystem::path SqliteExportService::BuildDatabasePath() const {
void SqliteExportService::InitializeSchema() const {
sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateLocationsTableSql,
"Failed to create SQLite locations table");
db_handle_, sqlite_export_service_internal::kCreateCitiesTableSql,
"Failed to create SQLite cities table");
sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateBreweriesTableSql,
"Failed to create SQLite breweries table");
sqlite_export_service_internal::ExecSql(
db_handle_,
sqlite_export_service_internal::kCreateBreweryAddressesTableSql,
"Failed to create SQLite brewery_addresses table");
sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateUsersTableSql,
"Failed to create SQLite users table");
sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateUserAddressesTableSql,
"Failed to create SQLite user_addresses table");
}
void SqliteExportService::PrepareStatements() {
insert_location_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertLocationSql,
"Failed to prepare SQLite location insert statement");
insert_city_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertCitySql,
"Failed to prepare SQLite city insert statement");
insert_brewery_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertBrewerySql,
"Failed to prepare SQLite brewery insert statement");
insert_brewery_address_stmt_ =
sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertBreweryAddressSql,
"Failed to prepare SQLite brewery address insert statement");
insert_user_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertUserSql,
"Failed to prepare SQLite user insert statement");
insert_user_address_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertUserAddressSql,
"Failed to prepare SQLite user address insert statement");
}
void SqliteExportService::RollbackAndCloseNoThrow() noexcept {
@@ -60,11 +73,13 @@ void SqliteExportService::RollbackAndCloseNoThrow() noexcept {
transaction_open_ = false;
}
insert_user_address_stmt_.reset();
insert_user_stmt_.reset();
insert_brewery_address_stmt_.reset();
insert_brewery_stmt_.reset();
insert_location_stmt_.reset();
insert_city_stmt_.reset();
db_handle_.reset();
location_cache_.clear();
city_cache_.clear();
}
void SqliteExportService::Initialize() {

160
tooling/pipeline/src/services/sqlite/process_record.cc
View File

@@ -1,10 +1,9 @@
/**
* @file services/sqlite/process_record.cc
* @brief SqliteExportService::ProcessRecord() implementation
* and the shared location-resolution helper.
* and the shared city-resolution helper.
*/
#include <iomanip>
#include <sstream>
#include <stdexcept>
#include <string>
@@ -12,94 +11,91 @@
#include "services/database/sqlite_export_service.h"
#include "services/database/sqlite_export_service_helpers.h"
constexpr int kLocationPrecision = 17;
std::string SqliteExportService::BuildLocationKey(const Location& location) {
std::string SqliteExportService::BuildCityKey(const City& city) {
std::ostringstream key_stream;
key_stream << location.city << '\n'
<< location.state_province << '\n'
<< location.iso3166_2 << '\n'
<< location.country << '\n'
<< location.iso3166_1 << '\n'
<< std::setprecision(kLocationPrecision) << location.latitude
<< '\n'
<< std::setprecision(kLocationPrecision) << location.longitude
<< '\n'
key_stream << city.city << '\n'
<< city.state_province << '\n'
<< city.iso3166_2 << '\n'
<< city.country << '\n'
<< city.iso3166_1 << '\n'
<< sqlite_export_service_internal::SerializeVector(
location.local_languages);
city.local_languages);
return key_stream.str();
}
sqlite3_int64 SqliteExportService::ResolveLocationId(const Location& location) {
const std::string location_key = BuildLocationKey(location);
const auto cached_location = location_cache_.find(location_key);
if (cached_location != location_cache_.end()) {
return cached_location->second;
sqlite3_int64 SqliteExportService::ResolveCityId(const City& city) {
const std::string city_key = BuildCityKey(city);
const auto cached_city = city_cache_.find(city_key);
if (cached_city != city_cache_.end()) {
return cached_city->second;
}
const std::string local_languages_json =
sqlite_export_service_internal::SerializeVector(location.local_languages);
sqlite_export_service_internal::SerializeVector(city.local_languages);
const std::string city_regex_json =
sqlite_export_service_internal::SerializeVector(
city.postal_code.city_regexes);
sqlite_export_service_internal::Bind(
insert_location_stmt_,
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kLocationCityBindIndex,
.value = location.city,
.action = "Failed to bind SQLite location city"});
.index = sqlite_export_service_internal::kCityCityBindIndex,
.value = city.city,
.action = "Failed to bind SQLite city name"});
sqlite_export_service_internal::Bind(
insert_location_stmt_,
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index =
sqlite_export_service_internal::kLocationStateProvinceBindIndex,
.value = location.state_province,
.action = "Failed to bind SQLite location state/province"});
.index = sqlite_export_service_internal::kCityStateProvinceBindIndex,
.value = city.state_province,
.action = "Failed to bind SQLite city state/province"});
sqlite_export_service_internal::Bind(
insert_location_stmt_,
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kLocationIso31662BindIndex,
.value = location.iso3166_2,
.action = "Failed to bind SQLite location ISO 3166-2 code"});
.index = sqlite_export_service_internal::kCityIso31662BindIndex,
.value = city.iso3166_2,
.action = "Failed to bind SQLite city ISO 3166-2 code"});
sqlite_export_service_internal::Bind(
insert_location_stmt_,
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kLocationCountryBindIndex,
.value = location.country,
.action = "Failed to bind SQLite location country"});
.index = sqlite_export_service_internal::kCityCountryBindIndex,
.value = city.country,
.action = "Failed to bind SQLite city country"});
sqlite_export_service_internal::Bind(
insert_location_stmt_,
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kLocationIso31661BindIndex,
.value = location.iso3166_1,
.action = "Failed to bind SQLite location ISO 3166-1 code"});
.index = sqlite_export_service_internal::kCityIso31661BindIndex,
.value = city.iso3166_1,
.action = "Failed to bind SQLite city ISO 3166-1 code"});
sqlite_export_service_internal::Bind(
insert_location_stmt_,
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kLocationLanguagesBindIndex,
.index = sqlite_export_service_internal::kCityLanguagesBindIndex,
.value = local_languages_json,
.action = "Failed to bind SQLite location languages"});
.action = "Failed to bind SQLite city languages"});
sqlite_export_service_internal::Bind(
insert_location_stmt_,
sqlite_export_service_internal::BoundParam{
.index = sqlite_export_service_internal::kLocationLatitudeBindIndex,
.value = location.latitude,
.action = "Failed to bind SQLite location latitude"});
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::
kCityPostalCodeCountryFormatRegexBindIndex,
.value = city.postal_code.country_format_regex,
.action = "Failed to bind SQLite city postal-code format regex"});
sqlite_export_service_internal::Bind(
insert_location_stmt_,
sqlite_export_service_internal::BoundParam{
.index = sqlite_export_service_internal::kLocationLongitudeBindIndex,
.value = location.longitude,
.action = "Failed to bind SQLite location longitude"});
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::
kCityPostalCodeCityRegexJsonBindIndex,
.value = city_regex_json,
.action = "Failed to bind SQLite city postal-code city regexes"});
sqlite_export_service_internal::StepStatement(
db_handle_, insert_location_stmt_,
"Failed to insert SQLite location row");
db_handle_, insert_city_stmt_, "Failed to insert SQLite city row");
const sqlite3_int64 location_id =
const sqlite3_int64 city_id =
sqlite_export_service_internal::LastInsertRowId(db_handle_);
location_cache_.emplace(location_key, location_id);
sqlite_export_service_internal::ResetStatement(insert_location_stmt_);
city_cache_.emplace(city_key, city_id);
sqlite_export_service_internal::ResetStatement(insert_city_stmt_);
return location_id;
return city_id;
}
uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
@@ -107,14 +103,7 @@ uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
throw std::runtime_error("SQLite export service is not initialized");
}
const sqlite3_int64 location_id = ResolveLocationId(brewery.location);
sqlite_export_service_internal::Bind(
insert_brewery_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index = sqlite_export_service_internal::kBreweryLocationIdBindIndex,
.value = location_id,
.action = "Failed to bind SQLite brewery location id"});
const sqlite3_int64 city_id = ResolveCityId(brewery.address.city);
sqlite_export_service_internal::Bind(
insert_brewery_stmt_,
@@ -149,7 +138,38 @@ uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
sqlite_export_service_internal::StepStatement(
db_handle_, insert_brewery_stmt_, "Failed to insert SQLite brewery row");
const sqlite3_int64 brewery_id =
sqlite_export_service_internal::LastInsertRowId(db_handle_);
sqlite_export_service_internal::ResetStatement(insert_brewery_stmt_);
return sqlite_export_service_internal::LastInsertRowId(db_handle_);
sqlite_export_service_internal::Bind(
insert_brewery_address_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index =
sqlite_export_service_internal::kBreweryAddressBreweryIdBindIndex,
.value = brewery_id,
.action = "Failed to bind SQLite brewery address brewery id"});
sqlite_export_service_internal::Bind(
insert_brewery_address_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index =
sqlite_export_service_internal::kBreweryAddressCityIdBindIndex,
.value = city_id,
.action = "Failed to bind SQLite brewery address city id"});
sqlite_export_service_internal::Bind(
insert_brewery_address_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::
kBreweryAddressPostalCodeBindIndex,
.value = brewery.address.postal_code,
.action = "Failed to bind SQLite brewery address postal code"});
sqlite_export_service_internal::StepStatement(
db_handle_, insert_brewery_address_stmt_,
"Failed to insert SQLite brewery address row");
sqlite_export_service_internal::ResetStatement(insert_brewery_address_stmt_);
return static_cast<uint64_t>(brewery_id);
}

39
tooling/pipeline/src/services/sqlite/process_user_record.cc
View File

@@ -13,14 +13,8 @@ uint64_t SqliteExportService::ProcessRecord(const UserRecord& user) {
throw std::runtime_error("SQLite export service is not initialized");
}
const sqlite3_int64 location_id = ResolveLocationId(user.location);
const sqlite3_int64 city_id = ResolveCityId(user.address.city);
sqlite_export_service_internal::Bind(
insert_user_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index = sqlite_export_service_internal::kUserLocationIdBindIndex,
.value = location_id,
.action = "Failed to bind SQLite user location id"});
sqlite_export_service_internal::Bind(
insert_user_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
@@ -73,7 +67,36 @@ uint64_t SqliteExportService::ProcessRecord(const UserRecord& user) {
sqlite_export_service_internal::StepStatement(
db_handle_, insert_user_stmt_, "Failed to insert SQLite user row");
const sqlite3_int64 user_id =
sqlite_export_service_internal::LastInsertRowId(db_handle_);
sqlite_export_service_internal::ResetStatement(insert_user_stmt_);
return sqlite_export_service_internal::LastInsertRowId(db_handle_);
sqlite_export_service_internal::Bind(
insert_user_address_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index = sqlite_export_service_internal::kUserAddressUserIdBindIndex,
.value = user_id,
.action = "Failed to bind SQLite user address user id"});
sqlite_export_service_internal::Bind(
insert_user_address_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index = sqlite_export_service_internal::kUserAddressCityIdBindIndex,
.value = city_id,
.action = "Failed to bind SQLite user address city id"});
sqlite_export_service_internal::Bind(
insert_user_address_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index =
sqlite_export_service_internal::kUserAddressPostalCodeBindIndex,
.value = user.address.postal_code,
.action = "Failed to bind SQLite user address postal code"});
sqlite_export_service_internal::StepStatement(
db_handle_, insert_user_address_stmt_,
"Failed to insert SQLite user address row");
sqlite_export_service_internal::ResetStatement(insert_user_address_stmt_);
return static_cast<uint64_t>(user_id);
}

23
tooling/pipeline/src/services/xeger/xeger.cc Normal file
View File

@@ -0,0 +1,23 @@
/**
* @file services/xeger/xeger.cc
* @brief Public entry point: parses a regex pattern into an AST and walks it
* to emit a random matching string ("xeger" -- "regex" backwards).
*
* See xeger_internal.h for the parser/AST/generator this delegates to.
*/
#include "services/xeger/xeger.h"
#include <random>
#include <string>
#include <string_view>
#include "services/xeger/xeger_internal.h"
std::string GenerateStringFromRegex(std::string_view pattern, std::mt19937& rng,
const int unbounded_repeat_cap) {
xeger_internal::Parser parser(pattern);
const xeger_internal::NodePtr ast = parser.Parse();
xeger_internal::XegerGenerator generator(rng, unbounded_repeat_cap);
return generator.Run(*ast);
}

110
tooling/pipeline/src/services/xeger/xeger_generator.cc Normal file
View File

@@ -0,0 +1,110 @@
/**
* @file services/xeger/xeger_generator.cc
* @brief Visitor that walks a regex AST and emits a random matching string.
*/
#include "services/xeger/xeger_internal.h"
#include <cassert>
#include <limits>
#include <random>
#include <stdexcept>
#include <string>
namespace xeger_internal {
XegerGenerator::XegerGenerator(std::mt19937& rng, const int unbounded_cap)
: rng_(rng), unbounded_cap_(unbounded_cap) {}
std::string XegerGenerator::Run(const Node& root) {
out_.clear();
root.Accept(*this);
return out_;
}
void XegerGenerator::Visit(const Literal& node) { out_ += node.ch; }
void XegerGenerator::Visit(const AnyChar&) {
static constexpr int kPrintableLow = ' '; // empty space - character 32
static constexpr int kPrintableHigh = '~'; // tilde - character 126
std::uniform_int_distribution<int> dist(kPrintableLow, kPrintableHigh);
// assert that the rand number is in the char range
bool inCharRange = dist.min() >= std::numeric_limits<char>::min() &&
dist.max() <= std::numeric_limits<char>::max();
assert(inCharRange && "Random number is out of char range");
out_ += static_cast<char>(dist(rng_));
}
void XegerGenerator::Visit(const CharClass& node) {
out_ += PickFromClass(node);
}
void XegerGenerator::Visit(const Concat& node) {
for (const NodePtr& part : node.parts) {
part->Accept(*this);
}
}
void XegerGenerator::Visit(const Alternation& node) {
std::uniform_int_distribution<size_t> dist(0, node.options.size() - 1);
node.options[dist(rng_)]->Accept(*this);
}
void XegerGenerator::Visit(const Repeat& node) {
const int lo = node.min;
const int hi = (node.max < 0) ? node.min + unbounded_cap_ : node.max;
std::uniform_int_distribution<int> dist(lo, hi);
const int count = dist(rng_);
for (int i = 0; i < count; ++i) {
node.child->Accept(*this);
}
}
bool XegerGenerator::InRanges(const CharClass& node, const char c) {
for (const auto& [lo, hi] : node.ranges) {
if (c >= lo && c <= hi) {
return true;
}
}
return false;
}
char XegerGenerator::PickFromClass(const CharClass& node) {
if (!node.negated) {
int total = 0;
for (const auto& [lo, hi] : node.ranges) {
total += hi - lo + 1;
}
std::uniform_int_distribution<int> dist(0, total - 1);
int k = dist(rng_);
for (const auto& [lo, hi] : node.ranges) {
const int width = hi - lo + 1;
if (k < width) {
return static_cast<char>(lo + k);
}
k -= width;
}
return node.ranges.back().second; // unreachable
}
// Negated class: draw random printable characters until one falls
// outside the forbidden set.
static constexpr int kPrintableLow = 32;
static constexpr int kPrintableHigh = 126;
static constexpr int kMaxAttempts = 1000;
std::uniform_int_distribution<int> dist(kPrintableLow, kPrintableHigh);
for (int attempt = 0; attempt < kMaxAttempts; ++attempt) {
const char c = static_cast<char>(dist(rng_));
if (!InRanges(node, c)) {
return c;
}
}
throw std::runtime_error(
"GenerateStringFromRegex: negated class rules out every printable "
"character");
}
} // namespace xeger_internal

307
tooling/pipeline/src/services/xeger/xeger_parser.cc Normal file
View File

@@ -0,0 +1,307 @@
/**
* @file services/xeger/xeger_parser.cc
* @brief Recursive-descent parser that turns a regex pattern into an AST.
*/
#include "services/xeger/xeger_internal.h"
#include <cctype>
#include <format>
#include <memory>
#include <stdexcept>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
namespace xeger_internal {
Parser::Parser(std::string_view pattern) : src_(StripAnchors(pattern)) {}
NodePtr Parser::Parse() {
NodePtr node = ParseAlternation();
if (!AtEnd()) {
throw std::runtime_error(
std::format("GenerateStringFromRegex: unexpected '{}' at position {}",
Peek(), pos_));
}
return node;
}
std::string Parser::StripAnchors(std::string_view pattern) {
if (!pattern.empty() && pattern.front() == kAnchorStart) {
pattern.remove_prefix(1);
}
const bool has_trailing_dollar =
!pattern.empty() && pattern.back() == kAnchorEnd &&
(pattern.size() < 2 || pattern[pattern.size() - 2] != kEscape);
if (has_trailing_dollar) {
pattern.remove_suffix(1);
}
return std::string(pattern);
}
bool Parser::AtEnd() const { return pos_ >= src_.size(); }
char Parser::Peek() const { return src_[pos_]; }
char Parser::Advance() { return src_[pos_++]; }
bool Parser::Match(const char c) {
if (!AtEnd() && Peek() == c) {
++pos_;
return true;
}
return false;
}
void Parser::Expect(const char c) {
if (!Match(c)) {
throw std::runtime_error(std::format(
"GenerateStringFromRegex: expected '{}' at position {}", c, pos_));
}
}
NodePtr Parser::ParseAlternation() {
std::vector<NodePtr> options;
options.push_back(ParseConcat());
while (Match(kAlternation)) {
options.push_back(ParseConcat());
}
if (options.size() == 1) {
return std::move(options.front());
}
auto alternation = std::make_unique<Alternation>();
alternation->options = std::move(options);
return alternation;
}
NodePtr Parser::ParseConcat() {
auto concat = std::make_unique<Concat>();
while (!AtEnd() && Peek() != kAlternation && Peek() != kGroupClose) {
concat->parts.push_back(ParseRepeat());
}
if (concat->parts.size() == 1) {
return std::move(concat->parts.front());
}
return concat;
}
NodePtr Parser::ParseRepeat() {
NodePtr atom = ParseAtom();
if (AtEnd()) {
return atom;
}
int lo = 0;
int hi = -1;
switch (Peek()) {
case kStar:
Advance();
lo = 0;
hi = -1;
break;
case kPlus:
Advance();
lo = 1;
hi = -1;
break;
case kOptional:
Advance();
lo = 0;
hi = 1;
break;
case kBraceOpen:
return ParseBrace(std::move(atom));
default:
return atom;
}
auto repeat = std::make_unique<Repeat>();
repeat->child = std::move(atom);
repeat->min = lo;
repeat->max = hi;
return repeat;
}
NodePtr Parser::ParseBrace(NodePtr atom) {
Expect(kBraceOpen);
std::string lower_digits;
std::string upper_digits;
bool has_comma = false;
while (!AtEnd() && std::isdigit(static_cast<unsigned char>(Peek())) != 0) {
lower_digits += Advance();
}
if (Match(kBraceComma)) {
has_comma = true;
while (!AtEnd() &&
std::isdigit(static_cast<unsigned char>(Peek())) != 0) {
upper_digits += Advance();
}
}
Expect(kBraceClose);
auto repeat = std::make_unique<Repeat>();
repeat->child = std::move(atom);
repeat->min = lower_digits.empty() ? 0 : std::stoi(lower_digits);
if (!has_comma) {
repeat->max = repeat->min;
} else if (upper_digits.empty()) {
repeat->max = -1;
} else {
repeat->max = std::stoi(upper_digits);
}
if (repeat->max != -1 && repeat->max < repeat->min) {
throw std::runtime_error(
"GenerateStringFromRegex: repetition max is smaller than min");
}
return repeat;
}
NodePtr Parser::ParseAtom() {
if (AtEnd()) {
throw std::runtime_error(
"GenerateStringFromRegex: pattern ended unexpectedly");
}
const char c = Peek();
switch (c) {
case kGroupOpen: {
Advance();
// Non-capturing group "(?:...)" -- capture semantics don't apply
// here (nothing back-references a group), so it's treated exactly
// like a plain group once the "?:" marker is consumed.
if (!AtEnd() && Peek() == kNonCapturingMarker) {
if (pos_ + 1 < src_.size() && src_[pos_ + 1] == kNonCapturingColon) {
pos_ += 2;
} else {
throw std::runtime_error(std::format(
"GenerateStringFromRegex: unsupported group syntax at "
"position {} (only plain '(...)' and non-capturing "
"'(?:...)' groups are supported)",
pos_));
}
}
NodePtr inner = ParseAlternation();
Expect(kGroupClose);
return inner;
}
case kClassOpen:
return ParseCharClass();
case kAnyChar:
Advance();
return std::make_unique<AnyChar>();
case kEscape:
Advance();
return ParseEscape();
case kStar:
case kPlus:
case kOptional:
case kBraceOpen:
throw std::runtime_error(std::format(
"GenerateStringFromRegex: quantifier with no preceding item at "
"position {}",
pos_));
default:
Advance();
return std::make_unique<Literal>(c);
}
}
NodePtr Parser::MakeClass(std::vector<std::pair<char, char>> ranges,
const bool negated) {
auto char_class = std::make_unique<CharClass>();
char_class->ranges = std::move(ranges);
char_class->negated = negated;
return char_class;
}
NodePtr Parser::ParseEscape() {
if (AtEnd()) {
throw std::runtime_error(
"GenerateStringFromRegex: a backslash at the very end has nothing "
"to escape");
}
const char c = Advance();
switch (c) {
case 'd':
return MakeClass({{'0', '9'}}, false);
case 'D':
return MakeClass({{'0', '9'}}, true);
case 'w':
return MakeClass({{'a', 'z'}, {'A', 'Z'}, {'0', '9'}, {'_', '_'}},
false);
case 'W':
return MakeClass({{'a', 'z'}, {'A', 'Z'}, {'0', '9'}, {'_', '_'}},
true);
case 's':
return MakeClass({{' ', ' '}, {'\t', '\t'}, {'\n', '\n'}, {'\r', '\r'}},
false);
case 'S':
return MakeClass({{' ', ' '}, {'\t', '\t'}, {'\n', '\n'}, {'\r', '\r'}},
true);
case 'n':
return std::make_unique<Literal>('\n');
case 't':
return std::make_unique<Literal>('\t');
case 'r':
return std::make_unique<Literal>('\r');
default:
// \. \* \( \\ etc. -- the literal character itself.
return std::make_unique<Literal>(c);
}
}
NodePtr Parser::ParseCharClass() {
Expect(kClassOpen);
auto char_class = std::make_unique<CharClass>();
if (Match(kClassNegate)) {
char_class->negated = true;
}
if (AtEnd()) {
throw std::runtime_error(
"GenerateStringFromRegex: character class '[' is never closed");
}
while (!AtEnd() && Peek() != kClassClose) {
char lo = ClassChar();
const bool is_range = Peek() == kClassRange && pos_ + 1 < src_.size() &&
src_[pos_ + 1] != kClassClose;
if (is_range) {
Advance();
char hi = ClassChar();
if (hi < lo) {
std::swap(lo, hi);
}
char_class->ranges.emplace_back(lo, hi);
} else {
char_class->ranges.emplace_back(lo, lo);
}
}
Expect(kClassClose);
if (char_class->ranges.empty()) {
throw std::runtime_error(
"GenerateStringFromRegex: empty character class []");
}
return char_class;
}
char Parser::ClassChar() {
const char c = Advance();
if (c != kEscape) {
return c;
}
if (AtEnd()) {
throw std::runtime_error(
"GenerateStringFromRegex: backslash at end of character class");
}
const char escaped = Advance();
switch (escaped) {
case 'n':
return '\n';
case 't':
return '\t';
case 'r':
return '\r';
default:
return escaped;
}
}
} // namespace xeger_internal