From 8e7ecf2fc39909d22565ffe8c7e3859bbc48d5cf Mon Sep 17 00:00:00 2001 From: Aaron Po Date: Sun, 19 Jul 2026 03:59:06 -0400 Subject: [PATCH] Pipeline: City rename, add address/postal-code generation, add xeger generation (#232) --- docs/pipeline/ETHICS-AND-KNOWN-ISSUES.md | 16 + tooling/pipeline/CMakeLists.txt | 15 + .../pipeline/includes/biergarten_pipeline.h | 6 + .../biergarten_pipeline_orchestrator.h | 7 +- .../includes/concurrency/bounded_channel.h | 2 - .../includes/data_generation/data_generator.h | 6 +- .../data_generation/llama_generator.h | 5 +- .../includes/data_generation/mock_generator.h | 15 +- .../includes/data_model/generated_models.h | 28 +- tooling/pipeline/includes/data_model/models.h | 48 +- .../curated_data/curated_data_service.h | 2 +- .../services/database/sqlite_export_service.h | 16 +- .../database/sqlite_statement_helpers.h | 122 +- .../services/datetime/date_time_provider.h | 3 + .../services/enrichment/enrichment_service.h | 4 +- .../services/enrichment/mock_enrichment.h | 2 +- .../services/enrichment/wikipedia_service.h | 2 +- .../includes/services/logging/log_entry.h | 2 +- .../postal_code/mock_postal_code_service.h | 79 ++ .../postal_code/postal_code_service.h | 35 + .../postal_code/xeger_postal_code_service.h | 32 + .../pipeline/includes/services/xeger/xeger.h | 38 + .../includes/services/xeger/xeger_internal.h | 201 +++ tooling/pipeline/locations.json | 1108 +++++++++++++++-- .../biergarten_pipeline_orchestrator.cc | 2 + .../generate_breweries.cc | 35 +- .../generate_users.cc | 19 +- .../log_results.cc | 18 +- .../query_cities_with_countries.cc | 8 +- .../biergarten_pipeline_orchestrator/run.cc | 2 +- .../data_generation/llama/generate_brewery.cc | 6 +- .../mock/deterministic_hash.cc | 4 +- .../data_generation/mock/generate_brewery.cc | 3 +- tooling/pipeline/src/main.cc | 20 +- .../curated_data/curated_json_data_service.cc | 32 +- .../curated_data/mock_curated_data_service.cc | 72 +- .../enrichment/wikipedia/get_summary.cc | 2 +- .../postal_code/xeger_postal_code_service.cc | 44 + .../pipeline/src/services/prompt_directory.cc | 11 +- .../pipeline/src/services/sqlite/finalize.cc | 6 +- .../helpers/sqlite_statement_helpers.cc | 3 +- .../src/services/sqlite/initialize.cc | 31 +- .../src/services/sqlite/process_record.cc | 160 +-- .../services/sqlite/process_user_record.cc | 39 +- tooling/pipeline/src/services/xeger/xeger.cc | 23 + .../src/services/xeger/xeger_generator.cc | 110 ++ .../src/services/xeger/xeger_parser.cc | 307 +++++ .../Database.Seed/Database.Seed.csproj | 9 +- .../Database/Database.Seed/DatabaseHelpers.cs | 54 + web/backend/Database/Database.Seed/ISeeder.cs | 19 - .../Database/Database.Seed/LocationSeeder.cs | 341 ----- web/backend/Database/Database.Seed/Program.cs | 106 -- .../Database/Database.Seed/UserSeeder.cs | 303 ----- web/backend/Database/Program.cs | 18 + .../Infrastructure.Jwt/JwtInfrastructure.cs | 2 +- .../Infrastructure.Sql.csproj | 2 +- 56 files changed, 2454 insertions(+), 1151 deletions(-) create mode 100644 tooling/pipeline/includes/services/postal_code/mock_postal_code_service.h create mode 100644 tooling/pipeline/includes/services/postal_code/postal_code_service.h create mode 100644 tooling/pipeline/includes/services/postal_code/xeger_postal_code_service.h create mode 100644 tooling/pipeline/includes/services/xeger/xeger.h create mode 100644 tooling/pipeline/includes/services/xeger/xeger_internal.h create mode 100644 tooling/pipeline/src/services/postal_code/xeger_postal_code_service.cc create mode 100644 tooling/pipeline/src/services/xeger/xeger.cc create mode 100644 tooling/pipeline/src/services/xeger/xeger_generator.cc create mode 100644 tooling/pipeline/src/services/xeger/xeger_parser.cc create mode 100644 web/backend/Database/Database.Seed/DatabaseHelpers.cs delete mode 100644 web/backend/Database/Database.Seed/ISeeder.cs delete mode 100644 web/backend/Database/Database.Seed/LocationSeeder.cs delete mode 100644 web/backend/Database/Database.Seed/Program.cs delete mode 100644 web/backend/Database/Database.Seed/UserSeeder.cs create mode 100644 web/backend/Database/Program.cs diff --git a/docs/pipeline/ETHICS-AND-KNOWN-ISSUES.md b/docs/pipeline/ETHICS-AND-KNOWN-ISSUES.md index febf030c..7892cf5c 100644 --- a/docs/pipeline/ETHICS-AND-KNOWN-ISSUES.md +++ b/docs/pipeline/ETHICS-AND-KNOWN-ISSUES.md @@ -18,6 +18,7 @@ low-resource language failures. - [Known Issues](#known-issues) - [Hallucinated Brewing Techniques](#hallucinated-brewing-techniques) - [Low-Resource Language Hallucination](#low-resource-language-hallucination) + - [Synthetic Postal Codes](#synthetic-postal-codes) --- @@ -286,6 +287,21 @@ Output sample: SQLite schema so downstream applications can filter or flag potentially hallucinated text by language tier. +### Synthetic Postal Codes + +When the pipeline is run without `--mocked`, `XegerPostalCodeService` generates +a fresh postal code per brewery/user address by reverse-matching one of the +city's curated postal-code regexes from `locations.json` (falling back to the +country format regex), rather than returning one of `MockPostalCodeService`'s +curated real-world examples. + +The generated code is **format-conformant by construction** — it always +satisfies the regex it was generated from — but it is not looked up from any +real postal registry. It does not correspond to a real address, business, or +resident, and should not be treated as accurate geocoding or delivery data. +This is consistent with the rest of the dataset: fixture data for a +proof-of-concept, not a source of truth. + --- ## Footnotes diff --git a/tooling/pipeline/CMakeLists.txt b/tooling/pipeline/CMakeLists.txt index d37bfeab..2716c490 100644 --- a/tooling/pipeline/CMakeLists.txt +++ b/tooling/pipeline/CMakeLists.txt @@ -111,6 +111,8 @@ if(NOT BIERGARTEN_MOCK_ONLY) ) FetchContent_MakeAvailable(llama-cpp) endif() + + find_package(OpenMP REQUIRED) endif() # spdlog @@ -208,6 +210,18 @@ target_sources(${PROJECT_NAME} PRIVATE src/services/enrichment/wikipedia/get_summary.cc ) +# --- services: postal_code --- +target_sources(${PROJECT_NAME} PRIVATE + src/services/postal_code/xeger_postal_code_service.cc +) + +# --- services: xeger --- +target_sources(${PROJECT_NAME} PRIVATE + src/services/xeger/xeger.cc + src/services/xeger/xeger_parser.cc + src/services/xeger/xeger_generator.cc +) + # --- services: sqlite --- target_sources(${PROJECT_NAME} PRIVATE src/services/sqlite/process_record.cc @@ -237,6 +251,7 @@ target_include_directories(${PROJECT_NAME} PRIVATE target_link_libraries(${PROJECT_NAME} PRIVATE $<$>:llama> + $<$>:OpenMP::OpenMP_CXX> boost::di Boost::json Boost::program_options diff --git a/tooling/pipeline/includes/biergarten_pipeline.h b/tooling/pipeline/includes/biergarten_pipeline.h index f9061fd9..6a66d5ed 100644 --- a/tooling/pipeline/includes/biergarten_pipeline.h +++ b/tooling/pipeline/includes/biergarten_pipeline.h @@ -58,6 +58,12 @@ #include "services/enrichment/mock_enrichment.h" #include "services/enrichment/wikipedia_service.h" +// --- services: postal_code --- +#include "services/postal_code/mock_postal_code_service.h" +#include "services/postal_code/postal_code_service.h" +#include "services/postal_code/xeger_postal_code_service.h" +#include "services/xeger/xeger.h" + // --- services: logging --- #include "services/logging/log_dispatcher.h" #include "services/logging/log_entry.h" diff --git a/tooling/pipeline/includes/biergarten_pipeline_orchestrator.h b/tooling/pipeline/includes/biergarten_pipeline_orchestrator.h index c65219a0..89dbc648 100644 --- a/tooling/pipeline/includes/biergarten_pipeline_orchestrator.h +++ b/tooling/pipeline/includes/biergarten_pipeline_orchestrator.h @@ -16,6 +16,7 @@ #include "services/database/export_service.h" #include "services/enrichment/enrichment_service.h" #include "services/logging/logger.h" +#include "services/postal_code/postal_code_service.h" /** * @brief Main orchestrator for the Biergarten data generation pipeline. @@ -34,6 +35,8 @@ class BiergartenPipelineOrchestrator { * @param exporter Database backend for persisting generated records. * @param curated_data_service Loads curated location, persona, and name * data used to seed generation. + * @param postal_code_service Resolves a postal code for a city from its + * curated postal-code data. * @param application_options CLI configuration and paths. */ BiergartenPipelineOrchestrator( @@ -42,6 +45,7 @@ class BiergartenPipelineOrchestrator { std::unique_ptr generator, std::unique_ptr exporter, std::unique_ptr curated_data_service, + std::unique_ptr postal_code_service, const ApplicationOptions& application_options); /** @@ -78,6 +82,7 @@ class BiergartenPipelineOrchestrator { */ std::unique_ptr exporter_; std::unique_ptr curated_data_service_; + std::unique_ptr postal_code_service_; ApplicationOptions application_options_; @@ -87,7 +92,7 @@ class BiergartenPipelineOrchestrator { * @return Vector of locations randomly sampled per * PipelineOptions::location_count. */ - std::vector QueryCitiesWithCountries(); + std::vector QueryLocations(); /** * @brief Generate breweries for enriched cities. diff --git a/tooling/pipeline/includes/concurrency/bounded_channel.h b/tooling/pipeline/includes/concurrency/bounded_channel.h index 402bfc1b..0c5c2e06 100644 --- a/tooling/pipeline/includes/concurrency/bounded_channel.h +++ b/tooling/pipeline/includes/concurrency/bounded_channel.h @@ -26,9 +26,7 @@ */ template class BoundedChannel { - // ------------------------------------------------------------------------- // Internal state — all access must be guarded by mutex_. - // ------------------------------------------------------------------------- std::queue queue_; diff --git a/tooling/pipeline/includes/data_generation/data_generator.h b/tooling/pipeline/includes/data_generation/data_generator.h index b3408e40..1f74713a 100644 --- a/tooling/pipeline/includes/data_generation/data_generator.h +++ b/tooling/pipeline/includes/data_generation/data_generator.h @@ -20,12 +20,10 @@ class DataGenerator { /** * @brief Generates brewery data for a location. * - * @param location Location data - * @param region_context Additional regional context text. + * @param city Enriched city the brewery is associated with. * @return Brewery generation result. */ - virtual BreweryResult GenerateBrewery(const Location& location, - const std::string& region_context) = 0; + virtual BreweryResult GenerateBrewery(const EnrichedCity& city) = 0; /** * @brief Generates a user profile grounded in a sampled name and persona. diff --git a/tooling/pipeline/includes/data_generation/llama_generator.h b/tooling/pipeline/includes/data_generation/llama_generator.h index 7d4dadb9..c0754ddd 100644 --- a/tooling/pipeline/includes/data_generation/llama_generator.h +++ b/tooling/pipeline/includes/data_generation/llama_generator.h @@ -51,12 +51,11 @@ class LlamaGenerator final : public DataGenerator { /** * @brief Generates brewery data for a specific location. * - * @param location Location object. + * @param location City object. * @param region_context Additional regional context. * @return Generated brewery result. */ - BreweryResult GenerateBrewery(const Location& location, - const std::string& region_context) override; + BreweryResult GenerateBrewery(const EnrichedCity&) override; /** * @brief Generates a user profile grounded in a sampled name and persona. diff --git a/tooling/pipeline/includes/data_generation/mock_generator.h b/tooling/pipeline/includes/data_generation/mock_generator.h index 62467d81..7d090372 100644 --- a/tooling/pipeline/includes/data_generation/mock_generator.h +++ b/tooling/pipeline/includes/data_generation/mock_generator.h @@ -20,20 +20,15 @@ class MockGenerator final : public DataGenerator { /** * @brief Generates deterministic brewery data for a location. * - * @param location City and country names. - * @param region_context Unused for mock generation. + * @param enriched_city * @return Generated brewery result. */ - BreweryResult GenerateBrewery(const Location& location, - const std::string& region_context) override; + BreweryResult GenerateBrewery(const EnrichedCity& enriched_city) override; /** * @brief Generates deterministic user data grounded in a sampled name and * persona. - * - * @param city Enriched city the user is associated with. - * @param persona Persona archetype. - * @param name Sampled first/last name, copied directly into the result. + * @return Generated user result. */ UserResult GenerateUser(const EnrichedCity& city, const UserPersona& persona, @@ -46,7 +41,7 @@ class MockGenerator final : public DataGenerator { * @param location City and country names. * @return Deterministic hash value. */ - static size_t DeterministicHash(const Location& location); + static size_t DeterministicHash(const City& location); /** * @brief Combines city, persona, and name into a stable hash value. @@ -56,7 +51,7 @@ class MockGenerator final : public DataGenerator { * @param name Sampled first/last name. * @return Deterministic hash value. */ - static size_t DeterministicHash(const Location& location, + static size_t DeterministicHash(const City& location, const UserPersona& persona, const Name& name); // Hash stride constants for deterministic distribution across fixed-size diff --git a/tooling/pipeline/includes/data_model/generated_models.h b/tooling/pipeline/includes/data_model/generated_models.h index 84a5fcf6..e47def6e 100644 --- a/tooling/pipeline/includes/data_model/generated_models.h +++ b/tooling/pipeline/includes/data_model/generated_models.h @@ -11,9 +11,7 @@ #include "data_model/models.h" -// ============================================================================ // Generation Output Models -// ============================================================================ /** * @brief Generated brewery payload. @@ -79,23 +77,39 @@ struct UserResult { float activity_weight{}; }; -// ============================================================================ // Pipeline Data Models -// ============================================================================ /** * @brief Enriched city data with Wikipedia context. */ struct EnrichedCity { - Location location; + City location; std::string region_context{}; }; +/** + * @brief A physical address: the city plus a concrete postal code selected + * from that city's curated postal-code examples. + */ +struct BreweryAddress { + City city; + std::string postal_code{}; +}; + +/** + * @brief A physical address for a generated user: the city plus a concrete + * postal code selected from that city's curated postal-code examples. + */ +struct UserAddress { + City city; + std::string postal_code{}; +}; + /** * @brief Helper struct to store generated brewery data. */ struct BreweryRecord { - Location location; + BreweryAddress address; BreweryResult brewery; }; @@ -107,7 +121,7 @@ struct BreweryRecord { * consumer can register real accounts from the pipeline's SQLite export. */ struct UserRecord { - Location location; + UserAddress address; UserResult user; std::string email{}; std::string date_of_birth{}; diff --git a/tooling/pipeline/includes/data_model/models.h b/tooling/pipeline/includes/data_model/models.h index 8ed7355b..25e7b976 100644 --- a/tooling/pipeline/includes/data_model/models.h +++ b/tooling/pipeline/includes/data_model/models.h @@ -21,14 +21,36 @@ class ILogger; namespace prog_opts = boost::program_options; -// ============================================================================ -// Location Models -// ============================================================================ +// City Models /** - * @brief Canonical location record for city-level generation. + * @brief Curated postal-code metadata for a city, mirroring the + * `postal_code` object in the locations dataset. */ -struct Location { +struct PostalCodeSpec { + /** + * @brief Permissive country-wide postal-code format regex, used as the + * validation fallback for a selected example. + */ + std::string country_format_regex{}; + + /** + * @brief City-specific postal-code regexes (more precise than the country + * format). + */ + std::vector city_regexes{}; + + /** + * @brief Concrete example postal codes for this city. The postal code + * service selects from these rather than synthesizing codes. + */ + std::vector examples{}; +}; + +/** + * @brief Canonical city record for city-level generation. + */ +struct City { std::string city{}; std::string state_province{}; @@ -50,19 +72,13 @@ struct Location { std::vector local_languages{}; /** - * @brief Latitude in decimal degrees. + * @brief Postal-code metadata (format regex, city regexes, and concrete + * examples) sourced directly from the curated locations dataset. */ - double latitude{}; - - /** - * @brief Longitude in decimal degrees. - */ - double longitude{}; + PostalCodeSpec postal_code{}; }; -// ============================================================================ // Name / Persona Models -// ============================================================================ /** * @brief A sampled first/last name pair, with the source forename's gender. @@ -131,9 +147,7 @@ struct UserPersona { std::vector style_affinities{}; }; -// ============================================================================ // Configuration Models -// ============================================================================ /** * @brief LLM sampling parameters. @@ -223,9 +237,7 @@ struct ApplicationOptions { PipelineOptions pipeline; }; -// ============================================================================ // Function Declarations -// ============================================================================ std::optional ParseArguments( const int argc, char** argv, std::shared_ptr logger = nullptr); diff --git a/tooling/pipeline/includes/services/curated_data/curated_data_service.h b/tooling/pipeline/includes/services/curated_data/curated_data_service.h index d6505ce3..aaadbbd9 100644 --- a/tooling/pipeline/includes/services/curated_data/curated_data_service.h +++ b/tooling/pipeline/includes/services/curated_data/curated_data_service.h @@ -15,7 +15,7 @@ using ForenameList = std::vector; using SurnameList = std::vector; -using LocationsList = std::vector; +using LocationsList = std::vector; using PersonasList = std::vector; using ForenamesByCountryMap = std::unordered_map; using SurnamesByCountryMap = std::unordered_map; diff --git a/tooling/pipeline/includes/services/database/sqlite_export_service.h b/tooling/pipeline/includes/services/database/sqlite_export_service.h index e5c49361..258e0213 100644 --- a/tooling/pipeline/includes/services/database/sqlite_export_service.h +++ b/tooling/pipeline/includes/services/database/sqlite_export_service.h @@ -45,27 +45,29 @@ class SqliteExportService final : public IExportService { void RollbackAndCloseNoThrow() noexcept; [[nodiscard]] std::filesystem::path BuildDatabasePath() const; - [[nodiscard]] static std::string BuildLocationKey(const Location& location); + [[nodiscard]] static std::string BuildCityKey(const City& city); /** - * @brief Returns the row id for @p location, inserting it first if it has + * @brief Returns the row id for @p city, inserting it first if it has * not already been seen during this run. * - * Shared by both ProcessRecord() overloads so breweries and users - * referencing the same location resolve to the same row. + * Shared by both ProcessRecord() overloads so brewery and user addresses + * referencing the same city resolve to the same row. */ - [[nodiscard]] sqlite3_int64 ResolveLocationId(const Location& location); + [[nodiscard]] sqlite3_int64 ResolveCityId(const City& city); std::unique_ptr date_time_provider_; std::filesystem::path output_path_; std::string run_timestamp_utc_; std::filesystem::path database_path_; SqliteDatabaseHandle db_handle_; - SqliteStatementHandle insert_location_stmt_; + SqliteStatementHandle insert_city_stmt_; SqliteStatementHandle insert_brewery_stmt_; + SqliteStatementHandle insert_brewery_address_stmt_; SqliteStatementHandle insert_user_stmt_; + SqliteStatementHandle insert_user_address_stmt_; bool transaction_open_ = false; - std::unordered_map location_cache_; + std::unordered_map city_cache_; }; #endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATABASE_SQLITE_EXPORT_SERVICE_H_ diff --git a/tooling/pipeline/includes/services/database/sqlite_statement_helpers.h b/tooling/pipeline/includes/services/database/sqlite_statement_helpers.h index b2143c4c..1cdc2365 100644 --- a/tooling/pipeline/includes/services/database/sqlite_statement_helpers.h +++ b/tooling/pipeline/includes/services/database/sqlite_statement_helpers.h @@ -17,9 +17,9 @@ namespace sqlite_export_service_internal { -inline constexpr std::string_view kCreateLocationsTableSql = R"sql( +inline constexpr std::string_view kCreateCitiesTableSql = R"sql( -CREATE TABLE IF NOT EXISTS locations ( +CREATE TABLE IF NOT EXISTS cities ( id INTEGER PRIMARY KEY AUTOINCREMENT, city TEXT NOT NULL, state_province TEXT NOT NULL, @@ -27,9 +27,9 @@ CREATE TABLE IF NOT EXISTS locations ( country TEXT NOT NULL, iso3166_1 TEXT NOT NULL, local_languages_json TEXT NOT NULL, - latitude REAL NOT NULL, - longitude REAL NOT NULL, - UNIQUE(city, state_province, iso3166_2, country, latitude, longitude) + postal_code_country_format_regex TEXT NOT NULL, + postal_code_city_regex_json TEXT NOT NULL, + UNIQUE(city, state_province, iso3166_2, country) ); )sql"; @@ -38,15 +38,29 @@ inline constexpr std::string_view kCreateBreweriesTableSql = R"sql( CREATE TABLE IF NOT EXISTS breweries ( id INTEGER PRIMARY KEY AUTOINCREMENT, - location_id INTEGER NOT NULL, name_en TEXT NOT NULL, description_en TEXT NOT NULL, name_local TEXT NOT NULL, - description_local TEXT NOT NULL, - FOREIGN KEY(location_id) REFERENCES locations(id) ON DELETE CASCADE + description_local TEXT NOT NULL ); -CREATE INDEX IF NOT EXISTS idx_breweries_location_id ON breweries(location_id); +)sql"; + +inline constexpr std::string_view kCreateBreweryAddressesTableSql = R"sql( + +CREATE TABLE IF NOT EXISTS brewery_addresses ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + brewery_id INTEGER NOT NULL, + city_id INTEGER NOT NULL, + postal_code TEXT NOT NULL, + FOREIGN KEY(brewery_id) REFERENCES breweries(id) ON DELETE CASCADE, + FOREIGN KEY(city_id) REFERENCES cities(id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_brewery_addresses_brewery_id + ON brewery_addresses(brewery_id); +CREATE INDEX IF NOT EXISTS idx_brewery_addresses_city_id + ON brewery_addresses(city_id); )sql"; @@ -54,7 +68,6 @@ inline constexpr std::string_view kCreateUsersTableSql = R"sql( CREATE TABLE IF NOT EXISTS users ( id INTEGER PRIMARY KEY AUTOINCREMENT, - location_id INTEGER NOT NULL, first_name TEXT NOT NULL, last_name TEXT NOT NULL, gender TEXT NOT NULL, @@ -62,40 +75,61 @@ CREATE TABLE IF NOT EXISTS users ( bio TEXT NOT NULL, activity_weight REAL NOT NULL, email TEXT NOT NULL UNIQUE, - date_of_birth TEXT NOT NULL, - FOREIGN KEY(location_id) REFERENCES locations(id) ON DELETE CASCADE + date_of_birth TEXT NOT NULL ); -CREATE INDEX IF NOT EXISTS idx_users_location_id ON users(location_id); - )sql"; -inline constexpr std::string_view kInsertLocationSql = R"sql( -INSERT INTO locations ( +inline constexpr std::string_view kCreateUserAddressesTableSql = R"sql( + +CREATE TABLE IF NOT EXISTS user_addresses ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + city_id INTEGER NOT NULL, + postal_code TEXT NOT NULL, + FOREIGN KEY(user_id) REFERENCES users(id) ON DELETE CASCADE, + FOREIGN KEY(city_id) REFERENCES cities(id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_user_addresses_user_id + ON user_addresses(user_id); +CREATE INDEX IF NOT EXISTS idx_user_addresses_city_id + ON user_addresses(city_id); + +)sql"; + +inline constexpr std::string_view kInsertCitySql = R"sql( +INSERT INTO cities ( city, state_province, iso3166_2, country, iso3166_1, local_languages_json, - latitude, - longitude + postal_code_country_format_regex, + postal_code_city_regex_json ) VALUES (?, ?, ?, ?, ?, ?, ?, ?); )sql"; inline constexpr std::string_view kInsertBrewerySql = R"sql( INSERT INTO breweries ( - location_id, name_en, description_en, name_local, description_local -) VALUES (?, ?, ?, ?, ?); +) VALUES (?, ?, ?, ?); +)sql"; + +inline constexpr std::string_view kInsertBreweryAddressSql = R"sql( +INSERT INTO brewery_addresses ( + brewery_id, + city_id, + postal_code +) VALUES (?, ?, ?); )sql"; inline constexpr std::string_view kInsertUserSql = R"sql( INSERT INTO users ( - location_id, first_name, last_name, gender, @@ -104,33 +138,45 @@ INSERT INTO users ( activity_weight, email, date_of_birth -) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?); +) VALUES (?, ?, ?, ?, ?, ?, ?, ?); +)sql"; + +inline constexpr std::string_view kInsertUserAddressSql = R"sql( +INSERT INTO user_addresses ( + user_id, + city_id, + postal_code +) VALUES (?, ?, ?); )sql"; // sqlite3_bind_*() parameter indices are 1-based, matching the "?" // placeholder order in the SQL above. -enum LocationBindIndex { - kLocationCityBindIndex = 1, - kLocationStateProvinceBindIndex, - kLocationIso31662BindIndex, - kLocationCountryBindIndex, - kLocationIso31661BindIndex, - kLocationLanguagesBindIndex, - kLocationLatitudeBindIndex, - kLocationLongitudeBindIndex, +enum CityBindIndex { + kCityCityBindIndex = 1, + kCityStateProvinceBindIndex, + kCityIso31662BindIndex, + kCityCountryBindIndex, + kCityIso31661BindIndex, + kCityLanguagesBindIndex, + kCityPostalCodeCountryFormatRegexBindIndex, + kCityPostalCodeCityRegexJsonBindIndex, }; enum BreweryBindIndex { - kBreweryLocationIdBindIndex = 1, - kBreweryEnglishNameBindIndex, + kBreweryEnglishNameBindIndex = 1, kBreweryEnglishDescriptionBindIndex, kBreweryLocalNameBindIndex, kBreweryLocalDescriptionBindIndex, }; +enum BreweryAddressBindIndex { + kBreweryAddressBreweryIdBindIndex = 1, + kBreweryAddressCityIdBindIndex, + kBreweryAddressPostalCodeBindIndex, +}; + enum UserBindIndex { - kUserLocationIdBindIndex = 1, - kUserFirstNameBindIndex, + kUserFirstNameBindIndex = 1, kUserLastNameBindIndex, kUserGenderBindIndex, kUserUsernameBindIndex, @@ -140,6 +186,12 @@ enum UserBindIndex { kUserDateOfBirthBindIndex, }; +enum UserAddressBindIndex { + kUserAddressUserIdBindIndex = 1, + kUserAddressCityIdBindIndex, + kUserAddressPostalCodeBindIndex, +}; + SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle, std::string_view sql, const char* action); diff --git a/tooling/pipeline/includes/services/datetime/date_time_provider.h b/tooling/pipeline/includes/services/datetime/date_time_provider.h index a5f8ce5d..070f4ad6 100644 --- a/tooling/pipeline/includes/services/datetime/date_time_provider.h +++ b/tooling/pipeline/includes/services/datetime/date_time_provider.h @@ -15,6 +15,9 @@ /** * @brief Interface for UTC timestamp providers. + * + * @todo Yagni, consider removing this and just use SystemDateTimeProvider + * directly */ class IDateTimeProvider { public: diff --git a/tooling/pipeline/includes/services/enrichment/enrichment_service.h b/tooling/pipeline/includes/services/enrichment/enrichment_service.h index 9c791372..34aa88b0 100644 --- a/tooling/pipeline/includes/services/enrichment/enrichment_service.h +++ b/tooling/pipeline/includes/services/enrichment/enrichment_service.h @@ -20,10 +20,10 @@ class IEnrichmentService { /** * @brief Resolves contextual enrichment for a location. * - * @param loc Location to enrich. + * @param loc City to enrich. * @return Context text, or an empty string if unavailable. */ - virtual std::string GetLocationContext(const Location& loc) = 0; + virtual std::string GetLocationContext(const City& loc) = 0; }; #endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ENRICHMENT_ENRICHMENT_SERVICE_H_ diff --git a/tooling/pipeline/includes/services/enrichment/mock_enrichment.h b/tooling/pipeline/includes/services/enrichment/mock_enrichment.h index 9a45b3f3..88bc39aa 100644 --- a/tooling/pipeline/includes/services/enrichment/mock_enrichment.h +++ b/tooling/pipeline/includes/services/enrichment/mock_enrichment.h @@ -15,7 +15,7 @@ */ class MockEnrichmentService final : public IEnrichmentService { public: - std::string GetLocationContext(const Location& /*loc*/) override { + std::string GetLocationContext(const City& /*loc*/) override { return {}; } }; diff --git a/tooling/pipeline/includes/services/enrichment/wikipedia_service.h b/tooling/pipeline/includes/services/enrichment/wikipedia_service.h index 1063aea8..e33b94a9 100644 --- a/tooling/pipeline/includes/services/enrichment/wikipedia_service.h +++ b/tooling/pipeline/includes/services/enrichment/wikipedia_service.h @@ -29,7 +29,7 @@ class WikipediaEnrichmentService final : public IEnrichmentService { /** * @brief Returns the Wikipedia-derived context for a location. */ - [[nodiscard]] std::string GetLocationContext(const Location& loc) override; + [[nodiscard]] std::string GetLocationContext(const City& loc) override; private: std::string FetchExtract(std::string_view query); diff --git a/tooling/pipeline/includes/services/logging/log_entry.h b/tooling/pipeline/includes/services/logging/log_entry.h index b6d6fba2..11131da3 100644 --- a/tooling/pipeline/includes/services/logging/log_entry.h +++ b/tooling/pipeline/includes/services/logging/log_entry.h @@ -51,7 +51,7 @@ enum class PipelinePhase { */ Startup, /** - * @brief Location/context enrichment (e.g. Wikipedia lookups). + * @brief City/context enrichment (e.g. Wikipedia lookups). */ Enrichment, /** diff --git a/tooling/pipeline/includes/services/postal_code/mock_postal_code_service.h b/tooling/pipeline/includes/services/postal_code/mock_postal_code_service.h new file mode 100644 index 00000000..87b05c4e --- /dev/null +++ b/tooling/pipeline/includes/services/postal_code/mock_postal_code_service.h @@ -0,0 +1,79 @@ +#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_ +#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_ + +/** + * @file services/postal_code/mock_postal_code_service.h + * @brief Data-driven postal code service that selects the first curated + * example for a city and validates it against the city's postal-code regex. + */ + +#include +#include +#include +#include + +#include "services/postal_code/postal_code_service.h" + +/** + * @brief Postal code service backed entirely by the curated locations + * dataset: it returns the city's first example postal code after checking it + * against the curated postal-code regex. + * + * A precise city regex is preferred when present; otherwise the permissive + * country format regex is used as the fallback. A city with no examples, or an + * example that fails validation, throws so the caller can skip the city. + */ +class MockPostalCodeService final : public IPostalCodeService { + public: + std::string GeneratePostalCode(const City& city) override { + const PostalCodeSpec& spec = city.postal_code; + + if (spec.examples.empty()) { + throw std::runtime_error(std::format( + "No postal-code examples available for city '{}'", city.city)); + } + + const std::string& candidate = spec.examples.front(); + + if (!MatchesAnyPattern(candidate, spec)) { + throw std::runtime_error( + std::format("Postal code '{}' for city '{}' does not match the " + "curated postal-code regex", + candidate, city.city)); + } + + return candidate; + } + + private: + /** + * @brief Returns true if @p candidate matches any city regex, or the + * country format regex when no city regex matches. When the spec carries no + * usable regex at all, validation is skipped (returns true). + */ + static bool MatchesAnyPattern(const std::string& candidate, + const PostalCodeSpec& spec) { + bool has_pattern = false; + + for (const std::string& pattern : spec.city_regexes) { + if (pattern.empty()) { + continue; + } + has_pattern = true; + if (std::regex_match(candidate, std::regex(pattern))) { + return true; + } + } + + if (!spec.country_format_regex.empty()) { + has_pattern = true; + if (std::regex_match(candidate, std::regex(spec.country_format_regex))) { + return true; + } + } + + return !has_pattern; + } +}; + +#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_ diff --git a/tooling/pipeline/includes/services/postal_code/postal_code_service.h b/tooling/pipeline/includes/services/postal_code/postal_code_service.h new file mode 100644 index 00000000..016cf369 --- /dev/null +++ b/tooling/pipeline/includes/services/postal_code/postal_code_service.h @@ -0,0 +1,35 @@ +#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_ +#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_ + +/** + * @file services/postal_code/postal_code_service.h + * @brief Abstraction for resolving a postal code for a city. + */ + +#include + +#include "data_model/models.h" + +/** + * @brief Interface for services that resolve a postal code for a city. + */ +class IPostalCodeService { + public: + IPostalCodeService() = default; + virtual ~IPostalCodeService() = default; + + IPostalCodeService(const IPostalCodeService&) = delete; + IPostalCodeService& operator=(const IPostalCodeService&) = delete; + IPostalCodeService(IPostalCodeService&&) = delete; + IPostalCodeService& operator=(IPostalCodeService&&) = delete; + + /** + * @brief Resolves a postal code for @p city. + * + * @param city City whose curated postal-code examples supply the result. + * @return A postal code string, or an empty string if none is available. + */ + virtual std::string GeneratePostalCode(const City& city) = 0; +}; + +#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_ diff --git a/tooling/pipeline/includes/services/postal_code/xeger_postal_code_service.h b/tooling/pipeline/includes/services/postal_code/xeger_postal_code_service.h new file mode 100644 index 00000000..6b38ec29 --- /dev/null +++ b/tooling/pipeline/includes/services/postal_code/xeger_postal_code_service.h @@ -0,0 +1,32 @@ +#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_XEGER_POSTAL_CODE_SERVICE_H_ +#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_XEGER_POSTAL_CODE_SERVICE_H_ + +/** + * @file services/postal_code/xeger_postal_code_service.h + * @brief Postal code service that generates a fresh, format-conformant code + * per call from the curated regex patterns, instead of returning a fixed + * example. + */ + +#include + +#include "services/postal_code/postal_code_service.h" + +/** + * @brief Postal code service using regex-driven generation (xeger). + * + * Picks among the city's curated `city_regexes` (falls back to + * `country_format_regex`), then generates a random string matching that pattern + * via GenerateStringFromRegex(). + */ +class XegerPostalCodeService final : public IPostalCodeService { + public: + XegerPostalCodeService(); + + std::string GeneratePostalCode(const City& city) override; + + private: + std::mt19937 rng_; +}; + +#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_XEGER_POSTAL_CODE_SERVICE_H_ diff --git a/tooling/pipeline/includes/services/xeger/xeger.h b/tooling/pipeline/includes/services/xeger/xeger.h new file mode 100644 index 00000000..678c1371 --- /dev/null +++ b/tooling/pipeline/includes/services/xeger/xeger.h @@ -0,0 +1,38 @@ +#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_XEGER_XEGER_H_ +#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_XEGER_XEGER_H_ + +#include +#include +#include + +/** + * @brief Generates a random string matching @p pattern. + * + * Supported syntax: + * - literals + * - `.` + * - alternation `|` + * - groups `(...)` and non-capturing groups `(?:...)` + * - quantifiers `*` `+` `?` `{n}` `{n,}` `{n,m}` + * - character classes `[...]` / `[^...]` + * - shorthand escapes `\d \D \w \W \s \S \n \t \r` + * + * Not supported: lookaround, backreferences, and lazy quantifiers. + * + * @param pattern Regular expression to generate a matching + * string for. + * @param rng Random source driving alternation/quantifier/ + * class choices. + * @param unbounded_repeat_cap Extra repeats allowed on top of the declared + * minimum for unbounded quantifiers + * (`*`, `+`, `{n,}`). + * + * @return A string satisfying `std::regex_match(result, std::regex(pattern))` + * for any @p pattern within the supported syntax subset. + * + * @throws std::runtime_error if @p pattern cannot be parsed. + */ +std::string GenerateStringFromRegex(std::string_view pattern, std::mt19937& rng, + int unbounded_repeat_cap = 8); + +#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_XEGER_XEGER_H_ diff --git a/tooling/pipeline/includes/services/xeger/xeger_internal.h b/tooling/pipeline/includes/services/xeger/xeger_internal.h new file mode 100644 index 00000000..b54d29b0 --- /dev/null +++ b/tooling/pipeline/includes/services/xeger/xeger_internal.h @@ -0,0 +1,201 @@ +#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_XEGER_XEGER_INTERNAL_H_ +#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_XEGER_XEGER_INTERNAL_H_ +/** + * @file services/xeger/xeger_internal.h + * @brief Declarations shared by the xeger parser and generator. + * + * 1. Parser Turns the pattern text into an AST (Literal, AnyChar, + * CharClass, Concat, Alternation, Repeat nodes). + * 2. Visitor Double-dispatch contract -- each AST-walking + * operation implements this. + * 3. XegerGenerator a Visitor that emits a random character wherever the + * AST says a character belongs, respecting alternation + * choices and repeat counts. + */ + +#include +#include +#include +#include +#include +#include +#include + +namespace xeger_internal { + +// Regex syntax characters + +constexpr char kAnchorStart = '^'; +constexpr char kAnchorEnd = '$'; +constexpr char kAlternation = '|'; +constexpr char kGroupOpen = '('; +constexpr char kGroupClose = ')'; +constexpr char kNonCapturingMarker = '?'; +constexpr char kNonCapturingColon = ':'; +constexpr char kClassOpen = '['; +constexpr char kClassClose = ']'; +constexpr char kClassNegate = '^'; +constexpr char kClassRange = '-'; +constexpr char kAnyChar = '.'; +constexpr char kEscape = '\\'; +constexpr char kStar = '*'; +constexpr char kPlus = '+'; +constexpr char kOptional = '?'; +constexpr char kBraceOpen = '{'; +constexpr char kBraceClose = '}'; +constexpr char kBraceComma = ','; + +// AST + +struct Literal; +struct AnyChar; +struct CharClass; +struct Concat; +struct Alternation; +struct Repeat; + +struct Visitor { + virtual void Visit(const Literal&) = 0; + virtual void Visit(const AnyChar&) = 0; + virtual void Visit(const CharClass&) = 0; + virtual void Visit(const Concat&) = 0; + virtual void Visit(const Alternation&) = 0; + virtual void Visit(const Repeat&) = 0; + virtual ~Visitor() = default; +}; + +struct Node { + virtual void Accept(Visitor&) const = 0; + virtual ~Node() = default; +}; + +using NodePtr = std::unique_ptr; + +template +struct Visitable : Node { + void Accept(Visitor& visitor) const override { + visitor.Visit(static_cast(*this)); + } +}; + +// One fixed character, e.g. 'a'. +struct Literal : Visitable { + char ch; + explicit Literal(const char c) : ch(c) {} +}; + +// The "." wildcard: any single printable character. +struct AnyChar : Visitable {}; + +// A set of allowed characters, stored as inclusive ranges. A single +// character 'x' is the range {'x', 'x'}. `negated` flips the meaning to +// "any character NOT listed" (i.e. `[^...]`). +struct CharClass : Visitable { + std::vector> ranges; + bool negated = false; +}; + +// "Do these children one after another." +struct Concat : Visitable { + std::vector parts; +}; + +// "Pick exactly one of these options at random" -- the `|` operator. +struct Alternation : Visitable { + std::vector options; +}; + +// "Repeat one child between min and max times." +// a* -> min 0, max infinite +// a+ -> min 1, max infinite +// a? -> min 0, max 1 +// a{2,5} -> min 2, max 5 +struct Repeat : Visitable { + NodePtr child; + unsigned int min = 0; + std::optional max; // std::nullopt means "no upper bound" +}; + +// Parser (recursive descent) +// +// Precedence, loosest to tightest binding: +// ParseAlternation ::= a|b|c +// ParseConcat ::= abc +// ParseRepeat ::= a* a{2,5} +// ParseAtom ::= a . [..] (..) \d + +class Parser { + public: + explicit Parser(std::string_view pattern); + + NodePtr Parse(); + + private: + std::string src_; + size_t pos_ = 0; + + // Strips anchors (leading '^' and trailing '$') as they are not needed for + // generation. + static std::string StripAnchors(std::string_view pattern); + + [[nodiscard]] bool AtEnd() const; + [[nodiscard]] char Peek() const; + char Advance(); + bool Match(char c); + void Expect(char c); + + // Level 1: alternation -- ( concat | concat | concat ) + NodePtr ParseAlternation(); + + // Level 2: concatenation -- a run of items back to back + NodePtr ParseConcat(); + + // Level 3: an atom plus an optional quantifier + NodePtr ParseRepeat(); + + // Curly-brace quantifiers: {n}, {n,}, {n,m} + NodePtr ParseBrace(NodePtr atom); + + // Level 4: an atom -- the smallest standalone unit + NodePtr ParseAtom(); + + static NodePtr MakeClass(std::vector> ranges, + bool negated); + + NodePtr ParseEscape(); + + NodePtr ParseCharClass(); + + // Reads one character inside [ ], turning \] \n etc. into the real + // character they represent. + char ClassChar(); +}; + +// Generator -- a Visitor that emits a random matching string + +class XegerGenerator final : public Visitor { + public: + XegerGenerator(std::mt19937& rng, int unbounded_cap); + + std::string Run(const Node& root); + + void Visit(const Literal& node) override; + void Visit(const AnyChar&) override; + void Visit(const CharClass& node) override; + void Visit(const Concat& node) override; + void Visit(const Alternation& node) override; + void Visit(const Repeat& node) override; + + private: + std::mt19937& rng_; + int unbounded_cap_; + std::string out_; + + static bool InRanges(const CharClass& node, char c); + + char PickFromClass(const CharClass& node); +}; + +} // namespace xeger_internal + +#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_XEGER_XEGER_INTERNAL_H_ diff --git a/tooling/pipeline/locations.json b/tooling/pipeline/locations.json index 2b216734..e0d37e51 100644 --- a/tooling/pipeline/locations.json +++ b/tooling/pipeline/locations.json @@ -7,7 +7,17 @@ "iso3166_1": "ZA", "latitude": -33.9249, "longitude": 18.4241, - "local_languages": ["af", "en", "xh"] + "local_languages": ["af", "en", "xh"], + "postal_code": { + "country_format_regex": "^\\d{4}$", + "city_regex": [ + "^(?:7(?:10[03]|4(?:[26][05]|0[04-7]|3[05-79]|4[0-4679]|5[0156]|8[09]|9[0139])|5(?:[56][01]|0[013-7]|25|3[0235-9]|40|8[12])|7(?:[23]5|0[058]|4[05]|5[0-256]|6[02-4679]|7[09]|8[02-9]|9[08])|8(?:[26]4|0[0-2568]|48|72|8[08])|9(?:[12]5|[59]0|3[57]|4[156]|6[56]|7[4-9]))|8(?:0(?:[246]0|0[0-3578]|1[0278]|5[01])|160))$" + ], + "examples": ["7100", "7103", "7400"], + "source": "derived from GeoNames postal data (radius, r=25km)", + "codes_matched": 135, + "notes": "Cape Town metro spans roughly 7000-8099; 7xxx also covers parts of the wider Western Cape. Enumerated from 135 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Johannesburg", @@ -17,7 +27,17 @@ "iso3166_1": "ZA", "latitude": -26.2041, "longitude": 28.0473, - "local_languages": ["en", "zu", "st", "af"] + "local_languages": ["en", "zu", "st", "af"], + "postal_code": { + "country_format_regex": "^\\d{4}$", + "city_regex": [ + "^(?:1(?:7(?:0[08]|1[1256])|8(?:0[13-9]|1[14-689]|37|41|5[23]|6[35]))|2(?:0(?:[156]\\d|0[013-9]|2[0-468]|3[0-8]|4[0-47-9]|7[02-46-8]|8[12]|9[0-5])|1(?:[01]\\d|2[0-489]|3[0-24-79]|4[0-57-9]|5[1-9]|6[0-579]|7[01]|8[89]|9[0-8])))$" + ], + "examples": ["1700", "1708", "1711"], + "source": "derived from GeoNames postal data (radius, r=18km)", + "codes_matched": 178, + "notes": "Johannesburg street codes cluster in 2000-2199; PO Box codes differ. Enumerated from 178 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Durban", @@ -27,7 +47,17 @@ "iso3166_1": "ZA", "latitude": -29.8587, "longitude": 31.0218, - "local_languages": ["zu", "en"] + "local_languages": ["zu", "en"], + "postal_code": { + "country_format_regex": "^\\d{4}$", + "city_regex": [ + "^(?:36(?:0[02-57-9]|1[0-7]|2[014-69]|3[0145]|5[0-25])|4(?:0(?:0[014-8]|1[13-9]|2[1-9]|3[124-9]|5[0-36-9]|6\\d|7[0-24589]|8[023589]|9[0-48])|1(?:0[149]|1[0135]|3[378]|4[48]|53)|3(?:[02]1|10|5[09]|6[0-24])))$" + ], + "examples": ["3600", "3602", "3603"], + "source": "derived from GeoNames postal data (radius, r=20km)", + "codes_matched": 121, + "notes": "eThekwini/Durban street codes cluster in 4000-4199. Enumerated from 121 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Franschhoek", @@ -37,7 +67,15 @@ "iso3166_1": "ZA", "latitude": -33.9146, "longitude": 19.1198, - "local_languages": ["af", "en"] + "local_languages": ["af", "en"], + "postal_code": { + "country_format_regex": "^\\d{4}$", + "city_regex": ["^769[01]$"], + "examples": ["7690", "7691"], + "source": "derived from GeoNames postal data (name, r=15km)", + "codes_matched": 2, + "notes": "Single town code. Enumerated from 2 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Nairobi", @@ -47,7 +85,17 @@ "iso3166_1": "KE", "latitude": -1.2921, "longitude": 36.8219, - "local_languages": ["sw", "en"] + "local_languages": ["sw", "en"], + "postal_code": { + "country_format_regex": "^\\d{5}$", + "city_regex": [ + "^00(?:[348]00|10[0-2]|2(?:1[03-69]|3[013569])|5(?:0\\d|1[0-35-9]|2[0-3])|6(?:0[013-79]|1[0-589]|2[0-35-7])|90[02356])$" + ], + "examples": ["00100", "00101", "00102"], + "source": "derived from GeoNames postal data (radius, r=20km)", + "codes_matched": 69, + "notes": "All Nairobi County codes begin 00; GPO is 00100. Range 00100-00999. Enumerated from 69 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Buenos Aires", @@ -57,7 +105,15 @@ "iso3166_1": "AR", "latitude": -34.6037, "longitude": -58.3816, - "local_languages": ["es-AR"] + "local_languages": ["es-AR"], + "postal_code": { + "country_format_regex": "^(?:[A-HJ-NP-TV-Z]\\d{4}[A-Z]{3}|\\d{4})$", + "city_regex": ["^C1[0-4]\\d{2}[A-Z]{3}$", "^1[0-4]\\d{2}$"], + "examples": ["C1425DKE", "1425"], + "source": "researched (no usable open dataset)", + "codes_matched": null, + "notes": "CABA uses CPA prefix 'C'. Legacy 4-digit form 1000-1499 still widely accepted." + } }, { "city": "Bariloche", @@ -67,7 +123,15 @@ "iso3166_1": "AR", "latitude": -41.1335, "longitude": -71.3103, - "local_languages": ["es-AR"] + "local_languages": ["es-AR"], + "postal_code": { + "country_format_regex": "^(?:[A-HJ-NP-TV-Z]\\d{4}[A-Z]{3}|\\d{4})$", + "city_regex": ["^840[09]$"], + "examples": ["8400", "8409"], + "source": "derived from GeoNames postal data (sub, r=25km)", + "codes_matched": 2, + "notes": "San Carlos de Bariloche legacy code 8400; CPA prefix 'R' (Rio Negro). Enumerated from 2 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Bogotá", @@ -77,7 +141,17 @@ "iso3166_1": "CO", "latitude": 4.711, "longitude": -74.0721, - "local_languages": ["es-CO"] + "local_languages": ["es-CO"], + "postal_code": { + "country_format_regex": "^\\d{6}$", + "city_regex": [ + "^11(?:0(?:[29][1-3]1|[36][12]1|[47][1-4]1|1[1-5]1|8[1-8]1)|1(?:[457]11|[23][12]1|[89][1-4]1|0[1-7]1|1(?:[1-4]1|[5-7][16])|6[1-3]1))$" + ], + "examples": ["110111", "110121", "110131"], + "source": "derived from GeoNames postal data (radius, r=20km)", + "codes_matched": 66, + "notes": "Bogota D.C. = DANE department code 11. Enumerated from 66 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Medellín", @@ -87,7 +161,15 @@ "iso3166_1": "CO", "latitude": 6.2442, "longitude": -75.5812, - "local_languages": ["es-CO"] + "local_languages": ["es-CO"], + "postal_code": { + "country_format_regex": "^\\d{6}$", + "city_regex": ["^0500(?:0[1-7]|1[0-8]|2\\d|3[0-7]|4[0-478])$"], + "examples": ["050001", "050002", "050003"], + "source": "derived from GeoNames postal data (name, r=25km)", + "codes_matched": 41, + "notes": "Antioquia = 05; Medellin zone = 050xxx. Wider metro uses 051xxx-055xxx. Enumerated from 41 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "São Paulo", @@ -97,7 +179,15 @@ "iso3166_1": "BR", "latitude": -23.5505, "longitude": -46.6333, - "local_languages": ["pt-BR"] + "local_languages": ["pt-BR"], + "postal_code": { + "country_format_regex": "^\\d{5}-?\\d{3}$", + "city_regex": ["^0[1-5]\\d{3}-?\\d{3}$", "^08[0-4]\\d{2}-?\\d{3}$"], + "examples": ["01310-100"], + "source": "researched (no usable open dataset)", + "codes_matched": null, + "notes": "Sao Paulo city CEP: 01000-000 to 05999-999, plus 08000-000 to 08499-999." + } }, { "city": "Curitiba", @@ -107,7 +197,15 @@ "iso3166_1": "BR", "latitude": -25.4284, "longitude": -49.2733, - "local_languages": ["pt-BR"] + "local_languages": ["pt-BR"], + "postal_code": { + "country_format_regex": "^\\d{5}-?\\d{3}$", + "city_regex": ["^8[0-2]\\d{3}-?\\d{3}$"], + "examples": ["80010-010"], + "source": "researched (no usable open dataset)", + "codes_matched": null, + "notes": "Curitiba CEP: 80000-000 to 82999-999." + } }, { "city": "Rio de Janeiro", @@ -117,7 +215,15 @@ "iso3166_1": "BR", "latitude": -22.9068, "longitude": -43.1729, - "local_languages": ["pt-BR"] + "local_languages": ["pt-BR"], + "postal_code": { + "country_format_regex": "^\\d{5}-?\\d{3}$", + "city_regex": ["^(?:2[0-2]\\d{3}|23[0-7]\\d{2})-?\\d{3}$"], + "examples": ["20040-020"], + "source": "researched (no usable open dataset)", + "codes_matched": null, + "notes": "Rio city CEP: 20000-000 to 23799-999." + } }, { "city": "Santiago", @@ -127,7 +233,17 @@ "iso3166_1": "CL", "latitude": -33.4489, "longitude": -70.6693, - "local_languages": ["es-CL"] + "local_languages": ["es-CL"], + "postal_code": { + "country_format_regex": "^\\d{7}$", + "city_regex": [ + "^(?:7(?:5[05]|63|75|8[15]|9[17])|8(?:[26]4|[48][26]|[59][048]|01|3[28]|7[08])|9(?:0[28]|1[26]|2[05]))\\d{4}$" + ], + "examples": ["7500000", "7550000", "7630000"], + "source": "derived from GeoNames postal data (radius, r=18km)", + "codes_matched": 31, + "notes": "Santiago Province spans approx. 7500000-9250000; 7xx = eastern communes, 8xx = central. Dataset is commune-level (nnn0000); 3-digit commune prefix + block. Enumerated from 31 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Valdivia", @@ -137,7 +253,15 @@ "iso3166_1": "CL", "latitude": -39.8142, "longitude": -73.2459, - "local_languages": ["es-CL"] + "local_languages": ["es-CL"], + "postal_code": { + "country_format_regex": "^\\d{7}$", + "city_regex": ["^509\\d{4}$"], + "examples": ["5090000"], + "source": "derived from GeoNames postal data (sub, r=20km)", + "codes_matched": 1, + "notes": "Los Rios region range approx. 5090000-5230000; Valdivia commune base code 5090000. Dataset is commune-level (nnn0000); 3-digit commune prefix + block. Enumerated from 1 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Lima", @@ -147,7 +271,17 @@ "iso3166_1": "PE", "latitude": -12.0464, "longitude": -77.0428, - "local_languages": ["es-PE"] + "local_languages": ["es-PE"], + "postal_code": { + "country_format_regex": "^\\d{5}$", + "city_regex": [ + "^15(?:0(?:0[1346-9]|1[1289]|2[1-46]|3[346-9]|4[6-9]|5[46-8]|6[3467]|7[2-469]|8[1-46-8]|9[346])|1(?:0[1-36-9]|1[23])|3(?:0[12467]|1[124]|2[6-8]|3[1-3])|4(?:[039][148]|[14][269]|[28][37]|53|79)|8(?:0[13469]|1[126-8]|2[489]|3[1467]|42))$" + ], + "examples": ["15001", "15003", "15004"], + "source": "derived from GeoNames postal data (radius, r=20km)", + "codes_matched": 111, + "notes": "Lima Region = prefix 15; metropolitan Lima districts sit in 15001-15099. Callao is 07xxx. Enumerated from 111 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Tokyo", @@ -157,7 +291,17 @@ "iso3166_1": "JP", "latitude": 35.6762, "longitude": 139.6503, - "local_languages": ["ja"] + "local_languages": ["ja"], + "postal_code": { + "country_format_regex": "^\\d{3}-?\\d{4}$", + "city_regex": [ + "^(?:1(?:[068]\\d|[14][0-6]|2[013-5]|3[0-24-7]|5[0-8]|7[013-9]|95)|20[1-46])-?\\d{4}$" + ], + "examples": ["100-0000", "100-0001", "100-0002"], + "source": "derived from GeoNames postal data (radius, r=20km)", + "codes_matched": 7876, + "notes": "Tokyo Metropolis: 100-xxxx to 208-xxxx. The 23 wards are 100-179. Dataset holds full 7-digit codes; compressed on the 3-digit area prefix. Enumerated from 7876 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Osaka", @@ -167,7 +311,15 @@ "iso3166_1": "JP", "latitude": 34.6937, "longitude": 135.5023, - "local_languages": ["ja"] + "local_languages": ["ja"], + "postal_code": { + "country_format_regex": "^\\d{3}-?\\d{4}$", + "city_regex": ["^5(?:[35]\\d|4[0-7])-?\\d{4}$"], + "examples": ["530-0000", "530-0001", "530-0002"], + "source": "derived from GeoNames postal data (radius, r=12km)", + "codes_matched": 1440, + "notes": "Osaka City: 530-xxxx to 559-xxxx. Wider Osaka Prefecture is 5xx. Dataset holds full 7-digit codes; compressed on the 3-digit area prefix. Enumerated from 1440 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Kyoto", @@ -177,7 +329,15 @@ "iso3166_1": "JP", "latitude": 35.0116, "longitude": 135.7681, - "local_languages": ["ja"] + "local_languages": ["ja"], + "postal_code": { + "country_format_regex": "^\\d{3}-?\\d{4}$", + "city_regex": ["^6(?:0[0-7]|1[0-256])-?\\d{4}$"], + "examples": ["600-0000", "600-8001", "600-8002"], + "source": "derived from GeoNames postal data (radius, r=12km)", + "codes_matched": 4614, + "notes": "Kyoto City: 600-xxxx to 616-xxxx. Dataset holds full 7-digit codes; compressed on the 3-digit area prefix. Enumerated from 4614 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Sapporo", @@ -187,7 +347,15 @@ "iso3166_1": "JP", "latitude": 43.0618, "longitude": 141.3545, - "local_languages": ["ja"] + "local_languages": ["ja"], + "postal_code": { + "country_format_regex": "^\\d{3}-?\\d{4}$", + "city_regex": ["^0(?:0[1-7]|6[0-5])-?\\d{4}$"], + "examples": ["001-0000", "001-0010", "001-0011"], + "source": "derived from GeoNames postal data (radius, r=20km)", + "codes_matched": 1373, + "notes": "Sapporo: 001-xxxx to 007-xxxx (north/west) and 060-xxxx to 065-xxxx (central/east). Dataset holds full 7-digit codes; compressed on the 3-digit area prefix. Enumerated from 1373 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Seoul", @@ -197,7 +365,17 @@ "iso3166_1": "KR", "latitude": 37.5665, "longitude": 126.978, - "local_languages": ["ko"] + "local_languages": ["ko"], + "postal_code": { + "country_format_regex": "^\\d{5}$", + "city_regex": [ + "^0(?:1[0-46-9]|2[0-24-8]|3[013-79]|4\\d|5[0-8]|6[0-35-9]|7[02-9]|8[0-35-8])\\d{2}$" + ], + "examples": ["01000", "01001", "01002"], + "source": "derived from GeoNames postal data (radius, r=18km)", + "codes_matched": 5628, + "notes": "Post-2015 5-digit system: Seoul = 01000-09999. Compressed on the 3-digit prefix. Enumerated from 5628 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Busan", @@ -207,7 +385,15 @@ "iso3166_1": "KR", "latitude": 35.1796, "longitude": 129.0756, - "local_languages": ["ko"] + "local_languages": ["ko"], + "postal_code": { + "country_format_regex": "^\\d{5}$", + "city_regex": ["^4(?:6[0235-79]|7[0-35-9]|8[0-57-9]|9[0-5])\\d{2}$"], + "examples": ["46002", "46018", "46027"], + "source": "derived from GeoNames postal data (radius, r=18km)", + "codes_matched": 2214, + "notes": "Busan = 46000-49999. Compressed on the 3-digit prefix. Enumerated from 2214 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Ho Chi Minh City", @@ -217,7 +403,15 @@ "iso3166_1": "VN", "latitude": 10.8231, "longitude": 106.6297, - "local_languages": ["vi"] + "local_languages": ["vi"], + "postal_code": { + "country_format_regex": "^\\d{5,6}$", + "city_regex": ["^7[0-4]\\d{3}$", "^7[0-4]\\d{4}$"], + "examples": ["70000", "700000"], + "source": "researched (no usable open dataset)", + "codes_matched": null, + "notes": "Current 5-digit system: HCMC = 70000-74999. Legacy/6-digit form (700000) still appears in some systems." + } }, { "city": "Hanoi", @@ -227,7 +421,15 @@ "iso3166_1": "VN", "latitude": 21.0285, "longitude": 105.8542, - "local_languages": ["vi"] + "local_languages": ["vi"], + "postal_code": { + "country_format_regex": "^\\d{5,6}$", + "city_regex": ["^1[0-4]\\d{3}$", "^1[0-4]\\d{4}$"], + "examples": ["10000", "100000"], + "source": "researched (no usable open dataset)", + "codes_matched": null, + "notes": "Hanoi = 10000-14999 (5-digit). Legacy 6-digit form 100000 still in circulation." + } }, { "city": "Da Nang", @@ -237,7 +439,15 @@ "iso3166_1": "VN", "latitude": 16.0544, "longitude": 108.2022, - "local_languages": ["vi"] + "local_languages": ["vi"], + "postal_code": { + "country_format_regex": "^\\d{5,6}$", + "city_regex": ["^5[0-5]\\d{3}$", "^5[0-5]\\d{4}$"], + "examples": ["50000", "550000"], + "source": "researched (no usable open dataset)", + "codes_matched": null, + "notes": "Sources conflict: Da Nang is cited as both 50000 and 55000. Pattern accepts 50000-55999 and the 6-digit legacy form." + } }, { "city": "Bangkok", @@ -247,7 +457,15 @@ "iso3166_1": "TH", "latitude": 13.7563, "longitude": 100.5018, - "local_languages": ["th"] + "local_languages": ["th"], + "postal_code": { + "country_format_regex": "^\\d{5}$", + "city_regex": ["^10(?:[4-8]00|1[0-7]0|2[024-69]0|3[0-3]0)$"], + "examples": ["10100", "10110", "10120"], + "source": "derived from GeoNames postal data (radius, r=20km)", + "codes_matched": 23, + "notes": "Bangkok = province code 10; codes run 10100-10900. Enumerated from 23 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Taipei", @@ -257,7 +475,15 @@ "iso3166_1": "TW", "latitude": 25.033, "longitude": 121.5654, - "local_languages": ["zh-TW"] + "local_languages": ["zh-TW"], + "postal_code": { + "country_format_regex": "^\\d{3}(?:\\d{2}|\\d{3})?$", + "city_regex": ["^1(?:0[0-9]|1[0-6])(?:\\d{2}|\\d{3})?$"], + "examples": ["100", "10058", "100013"], + "source": "researched (no usable open dataset)", + "codes_matched": null, + "notes": "Taipei City districts = 100-116. 3-digit is valid alone; 3+2 is legacy, 3+3 is current." + } }, { "city": "Beijing", @@ -267,7 +493,15 @@ "iso3166_1": "CN", "latitude": 39.9042, "longitude": 116.4074, - "local_languages": ["zh-CN"] + "local_languages": ["zh-CN"], + "postal_code": { + "country_format_regex": "^\\d{6}$", + "city_regex": ["^10[0-2]\\d{3}$"], + "examples": ["100000", "101100", "101200"], + "source": "derived from GeoNames postal data (radius, r=40km)", + "codes_matched": 11, + "notes": "Beijing municipality = 100000-102629. Dataset is city-level; compressed on the 3-digit prefix. Enumerated from 11 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Shanghai", @@ -277,7 +511,15 @@ "iso3166_1": "CN", "latitude": 31.2304, "longitude": 121.4737, - "local_languages": ["zh-CN"] + "local_languages": ["zh-CN"], + "postal_code": { + "country_format_regex": "^\\d{6}$", + "city_regex": ["^20[0-2]\\d{3}$"], + "examples": ["200000", "201100", "201300"], + "source": "derived from GeoNames postal data (radius, r=40km)", + "codes_matched": 10, + "notes": "Shanghai municipality = 200000-202699. Dataset is city-level; compressed on the 3-digit prefix. Enumerated from 10 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Bengaluru", @@ -287,7 +529,15 @@ "iso3166_1": "IN", "latitude": 12.9716, "longitude": 77.5946, - "local_languages": ["kn", "en"] + "local_languages": ["kn", "en"], + "postal_code": { + "country_format_regex": "^[1-9]\\d{5}$", + "city_regex": ["^560\\d{3}$"], + "examples": ["560001", "560002", "560003"], + "source": "derived from GeoNames postal data (field)", + "codes_matched": 106, + "notes": "Bengaluru urban PIN codes = 560xxx. Bengaluru Rural uses 561xxx/562xxx. Compressed on the 3-digit pin prefix. Enumerated from 106 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Singapore", @@ -297,7 +547,15 @@ "iso3166_1": "SG", "latitude": 1.3521, "longitude": 103.8198, - "local_languages": ["en", "zh", "ms", "ta"] + "local_languages": ["en", "zh", "ms", "ta"], + "postal_code": { + "country_format_regex": "^\\d{6}$", + "city_regex": ["^(?:[1-6]\\d|0[1-9]|7[0-35-9]|8[0-28]|91)\\d{4}$"], + "examples": ["018906", "018907", "018910"], + "source": "derived from GeoNames postal data (radius, r=30km)", + "codes_matched": 121135, + "notes": "Singapore is a single postal district; sectors 01-82 cover the whole island. Compressed on the 2-digit sector prefix. Enumerated from 121135 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Melbourne", @@ -307,7 +565,17 @@ "iso3166_1": "AU", "latitude": -37.8136, "longitude": 144.9631, - "local_languages": ["en-AU"] + "local_languages": ["en-AU"], + "postal_code": { + "country_format_regex": "^\\d{4}$", + "city_regex": [ + "^(?:3(?:0(?:[45]\\d|0[0-68]|1[0-35689]|2[0-35-9]|3[1-46-9]|6[0-25-8]|7[0-689]|8[1-9]|9[013-5])|1(?:[0248][1-9]|[39][0-4]|1[14]|5[0-2]|6[1-35-9]|7[0-24])|20[24-7]|428|752)|8(?:0(?:0[1-9]|1[0-2]|45|51|6[69]|7[01])|1(?:0[27]|11|20)|205|383|438|511))$" + ], + "examples": ["3000", "3001", "3002"], + "source": "derived from GeoNames postal data (radius, r=25km)", + "codes_matched": 179, + "notes": "Greater Melbourne approx. 3000-3207. Victoria as a whole is 3000-3999. Enumerated from 179 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Sydney", @@ -317,7 +585,17 @@ "iso3166_1": "AU", "latitude": -33.8688, "longitude": 151.2093, - "local_languages": ["en-AU"] + "local_languages": ["en-AU"], + "postal_code": { + "country_format_regex": "^\\d{4}$", + "city_regex": [ + "^(?:1(?:0(?:0[1-9]|10|2[0-35-9]|3\\d|4[0-6])|1(?:[2-9]\\d|0[015-9]|1[02-9])|2(?:0[1-3578]|3\\d|40|9[1-9])|3(?:[046]0|14|35|5[05])|4(?:[35][05]|[67][056]|0[15]|19|2[02-9]|4[015]|8[0145]|9[0359])|5(?:[18]5|6[05]|70|9[057])|6(?:[46]0|[78][05]|02|3[059]|5[58])|7(?:[09]0|[15][05]|[78]1|4[01]|65)|8(?:0[05]|1[19]|25|3[015]|60|7[45]|8[58]|91))|2(?:0(?:[2-46]\\d|[79][0-79]|0[0-246-9]|1[0-35-9]|5[0257-9]|8[4-9])|1(?:[1-39]\\d|0[0-49]|4[0-6]|5[0-24]|6[0-35]|7[23])|2(?:0[03-9]|1[0-46-9]|2\\d|3[014])))$" + ], + "examples": ["1001", "1002", "1003"], + "source": "derived from GeoNames postal data (radius, r=25km)", + "codes_matched": 424, + "notes": "Sydney metro approx. 2000-2249; 1000-1999 are Sydney PO Box / large-volume codes. Enumerated from 424 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Brisbane", @@ -327,7 +605,17 @@ "iso3166_1": "AU", "latitude": -27.4705, "longitude": 153.026, - "local_languages": ["en-AU"] + "local_languages": ["en-AU"], + "postal_code": { + "country_format_regex": "^\\d{4}$", + "city_regex": [ + "^(?:4(?:0(?:0\\d|1[0-478]|3[0-24-7]|5[13-59]|6[015-9]|7[02-8])|1(?:[05][1-9]|1\\d|2[0-37]|3[12]|6[0139]|7[0-489]))|90(?:0[0-2579]|1[035]))$" + ], + "examples": ["4000", "4001", "4002"], + "source": "derived from GeoNames postal data (radius, r=25km)", + "codes_matched": 99, + "notes": "Greater Brisbane approx. 4000-4207. Enumerated from 99 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Adelaide", @@ -337,7 +625,17 @@ "iso3166_1": "AU", "latitude": -34.9285, "longitude": 138.6007, - "local_languages": ["en-AU"] + "local_languages": ["en-AU"], + "postal_code": { + "country_format_regex": "^\\d{4}$", + "city_regex": [ + "^5(?:0(?:[14]\\d|[68][1-9]|0[015-9]|2[0-5]|3[1-57-9]|5[0-2]|7[0-6]|9[0-8])|1(?:0[6-9]|2[67]|3[1-46-9]|4[0-24]|5[0-24-689]|6[12])|8(?:0[01]|10|39)|9(?:42|50))$" + ], + "examples": ["5000", "5001", "5005"], + "source": "derived from GeoNames postal data (radius, r=20km)", + "codes_matched": 112, + "notes": "Adelaide metro approx. 5000-5199. Enumerated from 112 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Perth", @@ -347,7 +645,17 @@ "iso3166_1": "AU", "latitude": -31.9505, "longitude": 115.8605, - "local_languages": ["en-AU"] + "local_languages": ["en-AU"], + "postal_code": { + "country_format_regex": "^\\d{4}$", + "city_regex": [ + "^6(?:0(?:[56]\\d|0[013-9]|1[0-24-9]|2[0-79]|7[0167]|90)|1(?:[05]\\d|10|4[7-9]|6[02-46])|8(?:[79]2|0[039]|17|2[07]|3[0-27-9]|4\\d|50|65)|9(?:[18]\\d|[35][1-9]|0[0-79]|2[0-69]|4[0-7]|6[013-579]|7[09]|9[017]))$" + ], + "examples": ["6000", "6001", "6003"], + "source": "derived from GeoNames postal data (radius, r=25km)", + "codes_matched": 182, + "notes": "Perth metro approx. 6000-6199. Enumerated from 182 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Hobart", @@ -357,7 +665,15 @@ "iso3166_1": "AU", "latitude": -42.8821, "longitude": 147.3272, - "local_languages": ["en-AU"] + "local_languages": ["en-AU"], + "postal_code": { + "country_format_regex": "^\\d{4}$", + "city_regex": ["^7(?:0(?:0[0-24-9]|1[0-25-9]|2[13]|5[0-35])|170)$"], + "examples": ["7000", "7001", "7002"], + "source": "derived from GeoNames postal data (radius, r=15km)", + "codes_matched": 25, + "notes": "Hobart approx. 7000-7099. Enumerated from 25 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Wellington", @@ -367,7 +683,15 @@ "iso3166_1": "NZ", "latitude": -41.2865, "longitude": 174.7762, - "local_languages": ["en", "mi"] + "local_languages": ["en", "mi"], + "postal_code": { + "country_format_regex": "^\\d{4}$", + "city_regex": ["^60(?:1[12]|2[1-3]|3[57])$"], + "examples": ["6011", "6012", "6021"], + "source": "derived from GeoNames postal data (radius, r=12km)", + "codes_matched": 7, + "notes": "Wellington City = 6011-6037. Wider region extends to 5010-5999. Enumerated from 7 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Auckland", @@ -377,7 +701,17 @@ "iso3166_1": "NZ", "latitude": -36.8485, "longitude": 174.7633, - "local_languages": ["en", "mi"] + "local_languages": ["en", "mi"], + "postal_code": { + "country_format_regex": "^\\d{4}$", + "city_regex": [ + "^(?:0(?:6(?:0[024]|1[0248]|2[024679]|3[02]|4[0-5]|5[0-7]|6[0-3])|7(?:4\\d|5[0-8]|82|9[2-4])|8(?:1[04]|20|41|9[12])|9(?:30|43))|1(?:0(?:[47][12]|[56][0-2]|1[01]|2[1-6])|1(?:4[0-57-9]|5[01])|24[56]|3(?:4[024-9]|5[0-2])|44[0356]|54[1-6]|64[0-3]|74[0-6]|84[02])|2(?:0(?:1[02-468]|2[2-5])|1(?:0[2-5]|4[0-79]|5[013-9]|60)|24[0-3]|5(?:7[16]|80)))$" + ], + "examples": ["0600", "0602", "0604"], + "source": "derived from GeoNames postal data (radius, r=25km)", + "codes_matched": 169, + "notes": "Auckland is non-contiguous: 0600-0999 (north/west), 1010-1081 (central), 2010-2699 (south). Enumerated from 169 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Christchurch", @@ -387,7 +721,15 @@ "iso3166_1": "NZ", "latitude": -43.532, "longitude": 172.6306, - "local_languages": ["en", "mi"] + "local_languages": ["en", "mi"], + "postal_code": { + "country_format_regex": "^\\d{4}$", + "city_regex": ["^80(?:[46][12]|[58][1-3]|1[134]|2[2-5])$"], + "examples": ["8011", "8013", "8014"], + "source": "derived from GeoNames postal data (radius, r=15km)", + "codes_matched": 17, + "notes": "Christchurch City = 8011-8062. Canterbury region extends 7600-8999. Enumerated from 17 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Nelson", @@ -397,7 +739,15 @@ "iso3166_1": "NZ", "latitude": -41.2706, "longitude": 173.284, - "local_languages": ["en", "mi"] + "local_languages": ["en", "mi"], + "postal_code": { + "country_format_regex": "^\\d{4}$", + "city_regex": ["^70(?:1[01]|4[0-79])$"], + "examples": ["7010", "7011", "7040"], + "source": "derived from GeoNames postal data (radius, r=10km)", + "codes_matched": 11, + "notes": "Nelson City = 7010-7011 and 7040-7047. Enumerated from 11 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Munich", @@ -407,7 +757,17 @@ "iso3166_1": "DE", "latitude": 48.1351, "longitude": 11.582, - "local_languages": ["de"] + "local_languages": ["de"], + "postal_code": { + "country_format_regex": "^\\d{5}$", + "city_regex": [ + "^8(?:0(?:33[135-79]|469|53[89]|6(?:3[46-9]|8[679])|79[6-9]|80[1-579]|9(?:3[3579]|9[23579]))|1(?:[89]2[579]|24[1357-9]|3(?:69|7[13579])|47[5-79]|5(?:39|4[13579])|6(?:6[79]|7[13579])|73[579]))$" + ], + "examples": ["80331", "80333", "80335"], + "source": "derived from GeoNames postal data (name, r=20km)", + "codes_matched": 75, + "notes": "Munich = 80331-81929. Enumerated from 75 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Berlin", @@ -417,7 +777,17 @@ "iso3166_1": "DE", "latitude": 52.52, "longitude": 13.405, - "local_languages": ["de"] + "local_languages": ["de"], + "postal_code": { + "country_format_regex": "^\\d{5}$", + "city_regex": [ + "^1(?:0(?:[68]2[3579]|1(?:1[579]|7[89])|3(?:1[57-9]|6[579])|4[03][579]|5(?:5[13579]|8[579])|7(?:[07][79]|[18][13579]))|1011|2(?:0(?:4[3579]|5[13579]|99)|1(?:[06][13579]|5[79])|2(?:[47][79]|0[3579])|3(?:0[579]|4[79]|5[13579])|4(?:3[579]|59|8[79])|5(?:2[4679]|5[579]|8[79])|6(?:[17]9|2[1379]|8[13579]))|3(?:0(?:5[13579]|8[689])|1(?:2[579]|5[689]|8[79])|3(?:4[79]|5[13579])|4(?:[36][579]|0[3579])|5(?:[89][13579]|0[3579])|62[79])|4(?:0(?:5[023579]|89)|1(?:[02]9|[69][3579]|31)))$" + ], + "examples": ["10115", "10117", "10119"], + "source": "derived from GeoNames postal data (name, r=30km)", + "codes_matched": 182, + "notes": "Berlin = 10115-14199. Enumerated from 182 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Cologne", @@ -427,7 +797,17 @@ "iso3166_1": "DE", "latitude": 50.9375, "longitude": 6.9603, - "local_languages": ["de"] + "local_languages": ["de"], + "postal_code": { + "country_format_regex": "^\\d{5}$", + "city_regex": [ + "^5(?:0(?:6(?:6[78]|7[0246-9])|7(?:3[3579]|6[579])|8(?:2[3579]|5[89])|9(?:3[13579]|6[89]|9[679]))|1(?:06[13579]|1[04][3579]))$" + ], + "examples": ["50667", "50668", "50670"], + "source": "derived from GeoNames postal data (name, r=20km)", + "codes_matched": 45, + "notes": "Cologne = 50667-51149. Enumerated from 45 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Bamberg", @@ -437,7 +817,15 @@ "iso3166_1": "DE", "latitude": 49.8916, "longitude": 10.8916, - "local_languages": ["de"] + "local_languages": ["de"], + "postal_code": { + "country_format_regex": "^\\d{5}$", + "city_regex": ["^960(?:4[79]|5[02])$"], + "examples": ["96047", "96049", "96050"], + "source": "derived from GeoNames postal data (name, r=15km)", + "codes_matched": 4, + "notes": "Bamberg = 96047-96052. Enumerated from 4 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Brussels", @@ -447,7 +835,15 @@ "iso3166_1": "BE", "latitude": 50.8503, "longitude": 4.3517, - "local_languages": ["fr", "nl"] + "local_languages": ["fr", "nl"], + "postal_code": { + "country_format_regex": "^\\d{4}$", + "city_regex": ["^1(?:0(?:[02-79]0|8[0-3])|1[2-9]0|2[01]0)$"], + "examples": ["1000", "1020", "1030"], + "source": "derived from GeoNames postal data (radius, r=8km)", + "codes_matched": 22, + "notes": "Brussels-Capital Region = 1000-1210. Enumerated from 22 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Antwerp", @@ -457,7 +853,15 @@ "iso3166_1": "BE", "latitude": 51.2194, "longitude": 4.4025, - "local_languages": ["nl"] + "local_languages": ["nl"], + "postal_code": { + "country_format_regex": "^\\d{4}$", + "city_regex": ["^20(?:[02-6]0|18)$"], + "examples": ["2000", "2018", "2020"], + "source": "derived from GeoNames postal data (name, r=20km)", + "codes_matched": 7, + "notes": "City of Antwerp = 2000-2660; 2xxx also covers other Antwerp Province towns. Enumerated from 7 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Bruges", @@ -467,7 +871,15 @@ "iso3166_1": "BE", "latitude": 51.2093, "longitude": 3.2247, - "local_languages": ["nl"] + "local_languages": ["nl"], + "postal_code": { + "country_format_regex": "^\\d{4}$", + "city_regex": ["^8(?:[02]00|3[148]0)$"], + "examples": ["8000", "8200", "8310"], + "source": "derived from GeoNames postal data (radius, r=12km)", + "codes_matched": 5, + "notes": "Bruges uses discrete codes: 8000 (centre), 8200, 8310, 8340, 8380 (submunicipalities). Enumerated from 5 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "London", @@ -477,7 +889,17 @@ "iso3166_1": "GB", "latitude": 51.5074, "longitude": -0.1278, - "local_languages": ["en-GB"] + "local_languages": ["en-GB"], + "postal_code": { + "country_format_regex": "^[A-Z]{1,2}\\d[A-Z\\d]? ?\\d[A-Z]{2}$", + "city_regex": [ + "^(?:E(?:1W|C(?:[14][AMNRVY]|2[AMNPRVY]|3[AMNPV]))|SW1[EHPV-Y]|W(?:1[B-DF-HJKS-UW]|C(?:1[ABEHNRVX]|2[ABEHNR]))) ?\\d[A-Z]{2}$" + ], + "examples": ["E1W 1AA", "EC1A 1AA", "EC1M 1AA"], + "source": "derived from GeoNames postal data (name, r=25km)", + "codes_matched": 58, + "notes": "First pattern = inner London postcode areas; second = outer Greater London areas. Dataset is outward-code level; inward code appended. Enumerated from 58 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Bristol", @@ -487,7 +909,15 @@ "iso3166_1": "GB", "latitude": 51.4545, "longitude": -2.5879, - "local_languages": ["en-GB"] + "local_languages": ["en-GB"], + "postal_code": { + "country_format_regex": "^[A-Z]{1,2}\\d[A-Z\\d]? ?\\d[A-Z]{2}$", + "city_regex": ["^BS(?:[1256]|3[12459]?|4[01]) ?\\d[A-Z]{2}$"], + "examples": ["BS1 1AA", "BS2 1AA", "BS3 1AA"], + "source": "derived from GeoNames postal data (name, r=20km)", + "codes_matched": 12, + "notes": "BS postcode area (also covers parts of North Somerset / South Gloucestershire). Dataset is outward-code level; inward code appended. Enumerated from 12 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Edinburgh", @@ -497,7 +927,15 @@ "iso3166_1": "GB", "latitude": 55.9533, "longitude": -3.1883, - "local_languages": ["en-GB", "gd"] + "local_languages": ["en-GB", "gd"], + "postal_code": { + "country_format_regex": "^[A-Z]{1,2}\\d[A-Z\\d]? ?\\d[A-Z]{2}$", + "city_regex": ["^EH(?:[2-9]|1[12]?) ?\\d[A-Z]{2}$"], + "examples": ["EH1 1AA", "EH11 1AA", "EH12 1AA"], + "source": "derived from GeoNames postal data (name, r=20km)", + "codes_matched": 11, + "notes": "EH postcode area. Dataset is outward-code level; inward code appended. Enumerated from 11 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Glasgow", @@ -507,7 +945,15 @@ "iso3166_1": "GB", "latitude": 55.8642, "longitude": -4.2518, - "local_languages": ["en-GB", "gd"] + "local_languages": ["en-GB", "gd"], + "postal_code": { + "country_format_regex": "^[A-Z]{1,2}\\d[A-Z\\d]? ?\\d[A-Z]{2}$", + "city_regex": ["^G(?:[12][23]?|3[14]?|4[0-24]?|52?) ?\\d[A-Z]{2}$"], + "examples": ["G1 1AA", "G12 1AA", "G13 1AA"], + "source": "derived from GeoNames postal data (name, r=20km)", + "codes_matched": 16, + "notes": "G postcode area. Dataset is outward-code level; inward code appended. Enumerated from 16 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Prague", @@ -517,7 +963,17 @@ "iso3166_1": "CZ", "latitude": 50.0755, "longitude": 14.4378, - "local_languages": ["cs"] + "local_languages": ["cs"], + "postal_code": { + "country_format_regex": "^\\d{3} ?\\d{2}$", + "city_regex": [ + "^1(?:0[0-246-9]\\ 00|1[089]\\ 00|2[08]\\ 00|30\\ 00|4[0-37-9]\\ 00|5(?:[02489]\\ 00|5\\ (?:[23]1|00))|6[0-59]\\ 00|7[01]\\ 00|8[0-246]\\ 00|9(?:[3689]\\ 00|0\\ (?:00|1[1257])))$" + ], + "examples": ["100 00", "101 00", "102 00"], + "source": "derived from GeoNames postal data (radius, r=15km)", + "codes_matched": 52, + "notes": "Prague = 100 00 - 199 00. Enumerated from 52 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Pilsen", @@ -527,7 +983,15 @@ "iso3166_1": "CZ", "latitude": 49.7384, "longitude": 13.3736, - "local_languages": ["cs"] + "local_languages": ["cs"], + "postal_code": { + "country_format_regex": "^\\d{3} ?\\d{2}$", + "city_regex": ["^3(?:0[17]\\ 00|1[02]\\ 00)$"], + "examples": ["301 00", "307 00", "310 00"], + "source": "derived from GeoNames postal data (radius, r=10km)", + "codes_matched": 4, + "notes": "Plzen = 301 00 - 312 00. Enumerated from 4 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Amsterdam", @@ -537,7 +1001,17 @@ "iso3166_1": "NL", "latitude": 52.3676, "longitude": 4.9041, - "local_languages": ["nl"] + "local_languages": ["nl"], + "postal_code": { + "country_format_regex": "^\\d{4} ?[A-Z]{2}$", + "city_regex": [ + "^1(?:0(?:[157][1-9]|[29][1-8]|[34][1-7]|6\\d|8[1-367])|10[1-9]) ?[A-Z]{2}$" + ], + "examples": ["1011 AB", "1012 AB", "1013 AB"], + "source": "derived from GeoNames postal data (name, r=15km)", + "codes_matched": 81, + "notes": "Amsterdam = 1000-1109 numeric part. Dataset is 4-digit numeric; letter pair appended. Enumerated from 81 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Copenhagen", @@ -547,7 +1021,17 @@ "iso3166_1": "DK", "latitude": 55.6761, "longitude": 12.5683, - "local_languages": ["da"] + "local_languages": ["da"], + "postal_code": { + "country_format_regex": "^\\d{4}$", + "city_regex": [ + "^(?:1(?:8(?:[017]\\d|2[02-9]|35|5[0-7]|6[0-8])|9(?:[16][0-7]|0[0-689]|2[0-8]|5\\d|7[0-4]))|2000)$" + ], + "examples": ["1800", "1801", "1802"], + "source": "derived from GeoNames postal data (sub, r=12km)", + "codes_matched": 107, + "notes": "Copenhagen K = 1000-1499; Frederiksberg and surrounding districts = 1800-2500. Enumerated from 107 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Warsaw", @@ -557,7 +1041,17 @@ "iso3166_1": "PL", "latitude": 52.2297, "longitude": 21.0122, - "local_languages": ["pl"] + "local_languages": ["pl"], + "postal_code": { + "country_format_regex": "^\\d{2}-\\d{3}$", + "city_regex": [ + "^0(?:0\\-(?:[013-7]\\d|2[0-8]|8[0-79]|91)|1\\-(?:[13-57-9]\\d|[02][0-6]|6[0-57-9])|2\\-\\d\\d|3\\-(?:[0-79]\\d|8[0-46-9])|4\\-(?:[0-46-8]\\d|5[0-7]|9[05689]))\\d$" + ], + "examples": ["00-002", "00-003", "00-004"], + "source": "derived from GeoNames postal data (radius, r=15km)", + "codes_matched": 3661, + "notes": "Warsaw = 00-001 to 04-999. Compressed one digit short of full length. Enumerated from 3661 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Krakow", @@ -567,7 +1061,17 @@ "iso3166_1": "PL", "latitude": 50.0647, "longitude": 19.945, - "local_languages": ["pl"] + "local_languages": ["pl"], + "postal_code": { + "country_format_regex": "^\\d{2}-\\d{3}$", + "city_regex": [ + "^3(?:0\\-(?:[036]\\d|[27][0-589]|1[0-59]|4[0-49]|5[0-8]|8[0-69]|9[0469])|1\\-(?:[45][0-8]|[89]\\d|0[0-7]|1[0-6]|2[0-46-8]|3[0-79]|6[0-579]|7[0-2569]))\\d$" + ], + "examples": ["30-001", "30-002", "30-003"], + "source": "derived from GeoNames postal data (name, r=20km)", + "codes_matched": 1106, + "notes": "Krakow = 30-001 to 31-999. Compressed one digit short of full length. Enumerated from 1106 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Rome", @@ -577,7 +1081,15 @@ "iso3166_1": "IT", "latitude": 41.9028, "longitude": 12.4964, - "local_languages": ["it"] + "local_languages": ["it"], + "postal_code": { + "country_format_regex": "^\\d{5}$", + "city_regex": ["^001(?:[3-9][1-9]|1[89]|2[0-8])$"], + "examples": ["00118", "00119", "00120"], + "source": "derived from GeoNames postal data (name, r=20km)", + "codes_matched": 74, + "notes": "Rome = 00118-00199 (00120 = Vatican City, a separate state). Enumerated from 74 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Milan", @@ -587,7 +1099,15 @@ "iso3166_1": "IT", "latitude": 45.4642, "longitude": 9.19, - "local_languages": ["it"] + "local_languages": ["it"], + "postal_code": { + "country_format_regex": "^\\d{5}$", + "city_regex": ["^201(?:[2-5][1-9]|6[12])$"], + "examples": ["20121", "20122", "20123"], + "source": "derived from GeoNames postal data (name, r=15km)", + "codes_matched": 38, + "notes": "Milan city = 20121-20162. Enumerated from 38 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Barcelona", @@ -597,7 +1117,15 @@ "iso3166_1": "ES", "latitude": 41.3851, "longitude": 2.1734, - "local_languages": ["ca", "es"] + "local_languages": ["ca", "es"], + "postal_code": { + "country_format_regex": "^\\d{5}$", + "city_regex": ["^080(?:[1-3]\\d|0[1-9]|4[0-2]|7[015]|80)$"], + "examples": ["08001", "08002", "08003"], + "source": "derived from GeoNames postal data (name, r=15km)", + "codes_matched": 46, + "notes": "Barcelona city = 08001-08042. Province is 08xxx. Enumerated from 46 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Madrid", @@ -607,7 +1135,15 @@ "iso3166_1": "ES", "latitude": 40.4168, "longitude": -3.7038, - "local_languages": ["es"] + "local_languages": ["es"], + "postal_code": { + "country_format_regex": "^\\d{5}$", + "city_regex": ["^280(?:[1-4]\\d|0[1-9]|5[0-5]|7[01]|8[0235-7])$"], + "examples": ["28001", "28002", "28003"], + "source": "derived from GeoNames postal data (name, r=20km)", + "codes_matched": 63, + "notes": "Madrid city = 28001-28080. Community of Madrid is 28xxx. Enumerated from 63 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Paris", @@ -617,7 +1153,15 @@ "iso3166_1": "FR", "latitude": 48.8566, "longitude": 2.3522, - "local_languages": ["fr"] + "local_languages": ["fr"], + "postal_code": { + "country_format_regex": "^\\d{5}$", + "city_regex": ["^750(?:[01]\\d|20)$"], + "examples": ["75000", "75001", "75002"], + "source": "derived from GeoNames postal data (name, r=15km)", + "codes_matched": 21, + "notes": "Paris = 75001-75020, plus 75116 (a second code for the 16th arrondissement). Enumerated from 21 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Lyon", @@ -627,7 +1171,15 @@ "iso3166_1": "FR", "latitude": 45.764, "longitude": 4.8357, - "local_languages": ["fr"] + "local_languages": ["fr"], + "postal_code": { + "country_format_regex": "^\\d{5}$", + "city_regex": ["^6900\\d$"], + "examples": ["69000", "69001", "69002"], + "source": "derived from GeoNames postal data (name, r=15km)", + "codes_matched": 10, + "notes": "Lyon = 69001-69009 (nine arrondissements). Enumerated from 10 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Stockholm", @@ -637,7 +1189,17 @@ "iso3166_1": "SE", "latitude": 59.3293, "longitude": 18.0686, - "local_languages": ["sv"] + "local_languages": ["sv"], + "postal_code": { + "country_format_regex": "^\\d{3} ?\\d{2}$", + "city_regex": [ + "^1(?:0(?:[569]\\ \\d|[02]\\ [0-7]|1\\ [0-35]|3\\ [0-35-9]|4\\ [02-6]|7\\ [0-26-9])|1(?:[0-2478]\\ \\d|[35]\\ [02-9]|6\\ [0-46-9])|2(?:[17]\\ 2|0\\ [0-36-9]|5\\ 8|6\\ [0-278])|3(?:1\\ [12589]|5\\ 7)|4(?:1\\ [89]|5\\ 8)|6(?:[49]\\ [89]|1\\ 8|2\\ [18]|3\\ 9)|7(?:0\\ 8|1\\ [17]|2\\ [89]|3\\ [13-68])|82\\ 8|95\\ 8)\\d$" + ], + "examples": ["100 04", "100 05", "100 12"], + "source": "derived from GeoNames postal data (name, r=20km)", + "codes_matched": 1139, + "notes": "Stockholm County = 1xx xx. Compressed one digit short of full length. Enumerated from 1139 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Gothenburg", @@ -647,7 +1209,17 @@ "iso3166_1": "SE", "latitude": 57.7089, "longitude": 11.9746, - "local_languages": ["sv"] + "local_languages": ["sv"], + "postal_code": { + "country_format_regex": "^\\d{3} ?\\d{2}$", + "city_regex": [ + "^4(?:0(?:[89]\\ \\d|0\\ [1-79]|1\\ [12589]|2\\ [2-7]|3\\ [0-4]|4\\ [0238]|5\\ [0-57-9]|7\\ 0)|1(?:[26]\\ [024-9]|1\\ [0-489]|3\\ [0-2489]|4\\ [025-8]|5\\ \\d|7\\ [0-8]|8\\ [0-47]))\\d$" + ], + "examples": ["400 10", "400 12", "400 13"], + "source": "derived from GeoNames postal data (name, r=20km)", + "codes_matched": 755, + "notes": "Gothenburg = 400 00 - 425 99. Compressed one digit short of full length. Enumerated from 755 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Oslo", @@ -657,7 +1229,17 @@ "iso3166_1": "NO", "latitude": 59.9139, "longitude": 10.7522, - "local_languages": ["no"] + "local_languages": ["no"], + "postal_code": { + "country_format_regex": "^\\d{4}$", + "city_regex": [ + "^(?:0(?:[56][0-25-9]|0[0-5]|1\\d|2[013-8]|3[0-8]|4[0-24-9]|7[015-9]|8[04-9]|9[015-8])|1(?:[01][015-8]|2[015-9]))\\d$" + ], + "examples": ["0001", "0010", "0015"], + "source": "derived from GeoNames postal data (name, r=20km)", + "codes_matched": 631, + "notes": "Oslo = 0001-1295; the 0xxx block is the city proper. Compressed one digit short of full length. Enumerated from 631 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Dublin", @@ -667,7 +1249,15 @@ "iso3166_1": "IE", "latitude": 53.3498, "longitude": -6.2603, - "local_languages": ["en", "ga"] + "local_languages": ["en", "ga"], + "postal_code": { + "country_format_regex": "^(?:D6W|[AC-FHKNPRTV-Y]\\d{2}) ?[0-9AC-FHKNPRTV-Y]{4}$", + "city_regex": ["^D(?:0[1-9]|1[0-8]|2[024]|6W) ?[0-9AC-FHKNPRTV-Y]{4}$"], + "examples": ["D01 XY35", "D02 XY35", "D03 XY35"], + "source": "derived from GeoNames postal data (sub, r=25km)", + "codes_matched": 22, + "notes": "Dublin routing keys: D01-D18, D20, D22, D24 and the exception D6W. Note D19, D21, D23 are not issued. Dataset is routing-key level; unique identifier appended. Enumerated from 22 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Vienna", @@ -677,7 +1267,15 @@ "iso3166_1": "AT", "latitude": 48.2082, "longitude": 16.3738, - "local_languages": ["de-AT"] + "local_languages": ["de-AT"], + "postal_code": { + "country_format_regex": "^\\d{4}$", + "city_regex": ["^1(?:0[1-9]0|1\\d0|2[0-3]0)$"], + "examples": ["1010", "1020", "1030"], + "source": "derived from GeoNames postal data (field)", + "codes_matched": 23, + "notes": "Vienna = 1010-1230, always ending in 0 (district number x10). Enumerated from 23 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Zurich", @@ -687,7 +1285,17 @@ "iso3166_1": "CH", "latitude": 47.3769, "longitude": 8.5417, - "local_languages": ["de-CH"] + "local_languages": ["de-CH"], + "postal_code": { + "country_format_regex": "^\\d{4}$", + "city_regex": [ + "^80(?:0[0-68]|1[027]|2[247]|3[1246-8]|4[0-24-9]|5[0-3578]|6[034]|7[05]|8[178]|9[0-389])$" + ], + "examples": ["8000", "8001", "8002"], + "source": "derived from GeoNames postal data (name, r=12km)", + "codes_matched": 50, + "notes": "City of Zurich = 8001-8064. Enumerated from 50 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Tallinn", @@ -697,7 +1305,17 @@ "iso3166_1": "EE", "latitude": 59.437, "longitude": 24.7536, - "local_languages": ["et"] + "local_languages": ["et"], + "postal_code": { + "country_format_regex": "^\\d{5}$", + "city_regex": [ + "^(?:1(?:0(?:001|1(?:[2-4]\\d|1[1-9]|5[0-689])|3(?:1[1-9]|2[013])|41[1-6]|6(?:1[1-9]|2[01])|9(?:1[1-9]|2[0-2]))|1(?:[23]1[1-8]|101|41[1-7]|6(?:1[1-9]|2[0-5])|71[1-3]|91[1-4])|2(?:01[1-5]|11[1-3]|6(?:1[1-9]|20)|91[1-9])|3(?:4(?:1[1-9]|2[0-6])|5(?:1[1-9]|2[0-4])|6(?:1[1-9]|2\\d)|81[1267]|91[1-9])|5(?:0(?:[16]5|0[79]|2[69]|30|4[17-9]|5[59]|7[01]|9[4589])|1(?:6[15]|70|8[58]|9[039]))|9086)|7(?:4(?:0(?:0[18]|1[014-69]|20)|20[168])|5(?:3(?:0[1346-8]|1[02]|2[1-357]|31)|4(?:03|12)|511)|69(?:0[12]|1[26]|2[2-4])))$" + ], + "examples": ["10001", "10111", "10112"], + "source": "derived from GeoNames postal data (radius, r=12km)", + "codes_matched": 289, + "notes": "Tallinn = 10xxx-13xxx. Enumerated from 289 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Denver", @@ -707,7 +1325,17 @@ "iso3166_1": "US", "latitude": 39.7392, "longitude": -104.9903, - "local_languages": ["en-US"] + "local_languages": ["en-US"], + "postal_code": { + "country_format_regex": "^\\d{5}(?:-\\d{4})?$", + "city_regex": [ + "^802(?:[23]\\d|0[1-9]|1[0-24-9]|4[346-9]|5[0-2679]|6[0-6]|7[134]|81|9[01349])$" + ], + "examples": ["80201", "80202", "80203"], + "source": "derived from GeoNames postal data (name, r=30km)", + "codes_matched": 66, + "notes": "Denver = 80202-80249 plus 80290-80299. Enumerated from 66 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Portland", @@ -717,7 +1345,17 @@ "iso3166_1": "US", "latitude": 45.5152, "longitude": -122.6784, - "local_languages": ["en-US"] + "local_languages": ["en-US"], + "postal_code": { + "country_format_regex": "^\\d{5}(?:-\\d{4})?$", + "city_regex": [ + "^972(?:[59][0-468]|0[1-9]|1\\d|2[0-57-9]|3[0-3689]|4[02]|6[6-9]|8[0-36])$" + ], + "examples": ["97201", "97202", "97203"], + "source": "derived from GeoNames postal data (name, r=30km)", + "codes_matched": 60, + "notes": "Portland, Oregon = 972xx. Enumerated from 60 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "San Diego", @@ -727,7 +1365,17 @@ "iso3166_1": "US", "latitude": 32.7157, "longitude": -117.1611, - "local_languages": ["en-US", "es-US"] + "local_languages": ["en-US", "es-US"], + "postal_code": { + "country_format_regex": "^\\d{5}(?:-\\d{4})?$", + "city_regex": [ + "^921(?:0[1-9]|1[0-79]|2[0-469]|3[0-24-9]|4[02579]|5[02-589]|6[0135-9]|7[0-24-79]|8[267]|9[0-35-9])$" + ], + "examples": ["92101", "92102", "92103"], + "source": "derived from GeoNames postal data (name, r=30km)", + "codes_matched": 74, + "notes": "San Diego = 921xx. Enumerated from 74 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Asheville", @@ -737,7 +1385,15 @@ "iso3166_1": "US", "latitude": 35.5951, "longitude": -82.5515, - "local_languages": ["en-US"] + "local_languages": ["en-US"], + "postal_code": { + "country_format_regex": "^\\d{5}(?:-\\d{4})?$", + "city_regex": ["^288(?:0[1-6]|1[03-6])$"], + "examples": ["28801", "28802", "28803"], + "source": "derived from GeoNames postal data (name, r=25km)", + "codes_matched": 11, + "notes": "Asheville = 28801-28816. Enumerated from 11 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Grand Rapids", @@ -747,7 +1403,17 @@ "iso3166_1": "US", "latitude": 42.9634, "longitude": -85.6681, - "local_languages": ["en-US"] + "local_languages": ["en-US"], + "postal_code": { + "country_format_regex": "^\\d{5}(?:-\\d{4})?$", + "city_regex": [ + "^495(?:0[1-8]|1[024-68]|2[358]|3[04]|4[468]|55|60|88|99)$" + ], + "examples": ["49501", "49502", "49503"], + "source": "derived from GeoNames postal data (name, r=25km)", + "codes_matched": 26, + "notes": "Grand Rapids = 495xx. Enumerated from 26 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Chicago", @@ -757,7 +1423,17 @@ "iso3166_1": "US", "latitude": 41.8781, "longitude": -87.6298, - "local_languages": ["en-US", "es-US"] + "local_languages": ["en-US", "es-US"], + "postal_code": { + "country_format_regex": "^\\d{5}(?:-\\d{4})?$", + "city_regex": [ + "^60(?:6(?:0[1-9]|1\\d|2[0-689]|3[0-46-9]|4[0-79]|5[1-79]|6[014689]|7[03-578]|8[0-24-9]|9[013-79])|701)$" + ], + "examples": ["60601", "60602", "60603"], + "source": "derived from GeoNames postal data (name, r=30km)", + "codes_matched": 84, + "notes": "Chicago = 606xx-608xx. Enumerated from 84 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Seattle", @@ -767,7 +1443,17 @@ "iso3166_1": "US", "latitude": 47.6062, "longitude": -122.3321, - "local_languages": ["en-US"] + "local_languages": ["en-US"], + "postal_code": { + "country_format_regex": "^\\d{5}(?:-\\d{4})?$", + "city_regex": [ + "^981(?:[01][1-9]|2[124-79]|3[134689]|4[14-68]|5[458]|6[014-68]|7[04578]|8[158]|9[014589])$" + ], + "examples": ["98101", "98102", "98103"], + "source": "derived from GeoNames postal data (name, r=30km)", + "codes_matched": 59, + "notes": "Seattle = 981xx. Enumerated from 59 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Austin", @@ -777,7 +1463,17 @@ "iso3166_1": "US", "latitude": 30.2672, "longitude": -97.7431, - "local_languages": ["en-US", "es-US"] + "local_languages": ["en-US", "es-US"], + "postal_code": { + "country_format_regex": "^\\d{5}(?:-\\d{4})?$", + "city_regex": [ + "^7(?:33(?:01|44)|87(?:[1-356]\\d|0[1-589]|4[124-9]|7[2-489]|83|99))$" + ], + "examples": ["73301", "73344", "78701"], + "source": "derived from GeoNames postal data (name, r=30km)", + "codes_matched": 74, + "notes": "Austin = 787xx. Enumerated from 74 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Boston", @@ -787,7 +1483,17 @@ "iso3166_1": "US", "latitude": 42.3601, "longitude": -71.0589, - "local_languages": ["en-US"] + "local_languages": ["en-US"], + "postal_code": { + "country_format_regex": "^\\d{5}(?:-\\d{4})?$", + "city_regex": [ + "^02(?:1(?:[36]3|0[89]|1[0-8]|2[378]|9[69])|2(?:0[13-6]|1[0-257]|22|41|66|8[34]|9[378]))$" + ], + "examples": ["02108", "02109", "02110"], + "source": "derived from GeoNames postal data (name, r=25km)", + "codes_matched": 36, + "notes": "Boston = 021xx and 022xx. Enumerated from 36 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Philadelphia", @@ -797,7 +1503,17 @@ "iso3166_1": "US", "latitude": 39.9526, "longitude": -75.1652, - "local_languages": ["en-US"] + "local_languages": ["en-US"], + "postal_code": { + "country_format_regex": "^\\d{5}(?:-\\d{4})?$", + "city_regex": [ + "^19(?:0(?:19|9[239])|1(?:[2-4]\\d|0[1-9]|1[0-689]|5[0-5]|6[0-2]|7[0-35-9]|8[1-578]|9[0-7])|2(?:44|55))$" + ], + "examples": ["19019", "19092", "19093"], + "source": "derived from GeoNames postal data (name, r=30km)", + "codes_matched": 87, + "notes": "Philadelphia = 191xx. Enumerated from 87 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Brooklyn", @@ -807,7 +1523,15 @@ "iso3166_1": "US", "latitude": 40.6782, "longitude": -73.9442, - "local_languages": ["en-US", "es-US"] + "local_languages": ["en-US", "es-US"], + "postal_code": { + "country_format_regex": "^\\d{5}(?:-\\d{4})?$", + "city_regex": ["^112(?:[13]\\d|0[1-9]|2[0-689]|4[1-3579]|5[126])$"], + "examples": ["11201", "11202", "11203"], + "source": "derived from GeoNames postal data (name, r=25km)", + "codes_matched": 47, + "notes": "Brooklyn = 112xx. Enumerated from 47 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Milwaukee", @@ -817,7 +1541,17 @@ "iso3166_1": "US", "latitude": 43.0389, "longitude": -87.9065, - "local_languages": ["en-US"] + "local_languages": ["en-US"], + "postal_code": { + "country_format_regex": "^\\d{5}(?:-\\d{4})?$", + "city_regex": [ + "^532(?:0[1-9]|1\\d|2[0-8]|3[347]|59|6[378]|7[48]|88|9[035])$" + ], + "examples": ["53201", "53202", "53203"], + "source": "derived from GeoNames postal data (name, r=25km)", + "codes_matched": 41, + "notes": "Milwaukee = 532xx. Enumerated from 41 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Richmond", @@ -827,7 +1561,17 @@ "iso3166_1": "US", "latitude": 37.5407, "longitude": -77.436, - "local_languages": ["en-US"] + "local_languages": ["en-US"], + "postal_code": { + "country_format_regex": "^\\d{5}(?:-\\d{4})?$", + "city_regex": [ + "^23(?:173|2(?:1[89]|2[0-7]|3[024-7]|4[19]|50|6[019]|7[4689]|8[24-69]|9[0-3578]))$" + ], + "examples": ["23173", "23218", "23219"], + "source": "derived from GeoNames postal data (name, r=25km)", + "codes_matched": 39, + "notes": "Richmond = 232xx. Enumerated from 39 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Cincinnati", @@ -837,7 +1581,17 @@ "iso3166_1": "US", "latitude": 39.1031, "longitude": -84.512, - "local_languages": ["en-US"] + "local_languages": ["en-US"], + "postal_code": { + "country_format_regex": "^\\d{5}(?:-\\d{4})?$", + "city_regex": [ + "^45(?:2(?:[01][1-9]|[34]\\d|2[0-79]|5[0-58]|6[2-47-9]|7[013-57]|80|9[689])|999)$" + ], + "examples": ["45201", "45202", "45203"], + "source": "derived from GeoNames postal data (name, r=25km)", + "codes_matched": 71, + "notes": "Cincinnati = 452xx. Enumerated from 71 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "St. Louis", @@ -847,7 +1601,17 @@ "iso3166_1": "US", "latitude": 38.627, "longitude": -90.1994, - "local_languages": ["en-US"] + "local_languages": ["en-US"], + "postal_code": { + "country_format_regex": "^\\d{5}(?:-\\d{4})?$", + "city_regex": [ + "^631(?:[1-3]\\d|0[1-9]|4[013-7]|5[015-8]|6[034679]|7[17-9]|8[028]|9[579])$" + ], + "examples": ["63101", "63102", "63103"], + "source": "derived from GeoNames postal data (name, r=25km)", + "codes_matched": 68, + "notes": "St. Louis = 631xx. Enumerated from 68 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Tampa", @@ -857,7 +1621,17 @@ "iso3166_1": "US", "latitude": 27.9506, "longitude": -82.4572, - "local_languages": ["en-US", "es-US"] + "local_languages": ["en-US", "es-US"], + "postal_code": { + "country_format_regex": "^\\d{5}(?:-\\d{4})?$", + "city_regex": [ + "^336(?:0[1-9]|1\\d|2[0-69]|3[013-57]|4[67]|5[05]|6[0-4]|7[2-579]|8[0-24-9]|94)$" + ], + "examples": ["33601", "33602", "33603"], + "source": "derived from GeoNames postal data (name, r=30km)", + "codes_matched": 58, + "notes": "Tampa = 336xx. Enumerated from 58 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Minneapolis", @@ -867,7 +1641,17 @@ "iso3166_1": "US", "latitude": 44.9778, "longitude": -93.265, - "local_languages": ["en-US"] + "local_languages": ["en-US"], + "postal_code": { + "country_format_regex": "^\\d{5}(?:-\\d{4})?$", + "city_regex": [ + "^554(?:[1-4]\\d|0[1-9]|5[04589]|6[07]|7[02-489]|8[03-8])$" + ], + "examples": ["55401", "55402", "55403"], + "source": "derived from GeoNames postal data (name, r=25km)", + "codes_matched": 69, + "notes": "Minneapolis = 554xx. Enumerated from 69 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Burlington", @@ -877,7 +1661,15 @@ "iso3166_1": "US", "latitude": 44.4759, "longitude": -73.2121, - "local_languages": ["en-US"] + "local_languages": ["en-US"], + "postal_code": { + "country_format_regex": "^\\d{5}(?:-\\d{4})?$", + "city_regex": ["^0540[12568]$"], + "examples": ["05401", "05402", "05405"], + "source": "derived from GeoNames postal data (name, r=20km)", + "codes_matched": 5, + "notes": "Burlington = 054xx. Enumerated from 5 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Portland", @@ -887,7 +1679,15 @@ "iso3166_1": "US", "latitude": 43.6591, "longitude": -70.2568, - "local_languages": ["en-US"] + "local_languages": ["en-US"], + "postal_code": { + "country_format_regex": "^\\d{5}(?:-\\d{4})?$", + "city_regex": ["^041(?:0[1-49]|12|2[2-4])$"], + "examples": ["04101", "04102", "04103"], + "source": "derived from GeoNames postal data (name, r=25km)", + "codes_matched": 9, + "notes": "Portland, Maine = 041xx. Distinct from Portland, Oregon (972xx). Enumerated from 9 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Atlanta", @@ -897,7 +1697,17 @@ "iso3166_1": "US", "latitude": 33.749, "longitude": -84.388, - "local_languages": ["en-US"] + "local_languages": ["en-US"], + "postal_code": { + "country_format_regex": "^\\d{5}(?:-\\d{4})?$", + "city_regex": [ + "^3(?:03(?:0[1-9]|1\\d|2[0-24-9]|3[1246-9]|4[0-689]|5[03-578]|6[013489]|7[014578]|8[0458]|9[2468])|11(?:0[67]|26|3[169]|5[06]|9[2356]))$" + ], + "examples": ["30301", "30302", "30303"], + "source": "derived from GeoNames postal data (name, r=30km)", + "codes_matched": 82, + "notes": "Atlanta city = 303xx; the wider metro extends across 303xx-309xx. Enumerated from 82 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Toronto", @@ -907,7 +1717,17 @@ "iso3166_1": "CA", "latitude": 43.651, "longitude": -79.347, - "local_languages": ["en-CA"] + "local_languages": ["en-CA"], + "postal_code": { + "country_format_regex": "^[ABCEGHJ-NPRSTVXY]\\d[ABCEGHJ-NPRSTV-Z] ?\\d[ABCEGHJ-NPRSTV-Z]\\d$", + "city_regex": [ + "^M(?:1[BCEGHJ-NPR-TV-X]|2[HJ-NPR]|3[A-CHJ-N]|4[A-CEGHJ-NPR-TV-Y]|5[A-CEGHJ-NPR-TV-X]|6[A-CEGHJ-NPRS]|7[AY]|8[V-Z]|9[A-CL-NPRVW]) ?\\d[ABCEGHJ-NPRSTV-Z]\\d$" + ], + "examples": ["M1B 1A1", "M1C 1A1", "M1E 1A1"], + "source": "derived from GeoNames postal data (radius, r=30km)", + "codes_matched": 102, + "notes": "Toronto = the entire 'M' forward sortation area. Dataset is fsa-level; local delivery unit appended. Enumerated from 102 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Vancouver", @@ -917,7 +1737,17 @@ "iso3166_1": "CA", "latitude": 49.2827, "longitude": -123.1207, - "local_languages": ["en-CA"] + "local_languages": ["en-CA"], + "postal_code": { + "country_format_regex": "^[ABCEGHJ-NPRSTVXY]\\d[ABCEGHJ-NPRSTV-Z] ?\\d[ABCEGHJ-NPRSTV-Z]\\d$", + "city_regex": [ + "^V(?:5[K-NPR-TV-Z]|6[A-CEGHJ-NPRSZ]|7[GHJ-NPR-TV-Y]) ?\\d[ABCEGHJ-NPRSTV-Z]\\d$" + ], + "examples": ["V5K 1A1", "V5L 1A1", "V5M 1A1"], + "source": "derived from GeoNames postal data (sub, r=20km)", + "codes_matched": 43, + "notes": "Vancouver proper = V5/V6; V7 covers North/West Vancouver and Richmond. Dataset is fsa-level; local delivery unit appended. Enumerated from 43 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Montreal", @@ -927,7 +1757,17 @@ "iso3166_1": "CA", "latitude": 45.5017, "longitude": -73.5673, - "local_languages": ["fr-CA", "en-CA"] + "local_languages": ["fr-CA", "en-CA"], + "postal_code": { + "country_format_regex": "^[ABCEGHJ-NPRSTVXY]\\d[ABCEGHJ-NPRSTV-Z] ?\\d[ABCEGHJ-NPRSTV-Z]\\d$", + "city_regex": [ + "^H(?:[1-3][A-CEGHJ-NPR-TV-Z]|0M|4[A-CEGHJ-NPR-TV-XZ]|5[AB]|7[A-CEGHJ-NPR-TV-Y]|8[NPR-TYZ]|9[A-CGHJPRSW]) ?\\d[ABCEGHJ-NPRSTV-Z]\\d$" + ], + "examples": ["H0M 1A1", "H1A 1A1", "H1B 1A1"], + "source": "derived from GeoNames postal data (radius, r=25km)", + "codes_matched": 118, + "notes": "Montreal = the entire 'H' forward sortation area. Dataset is fsa-level; local delivery unit appended. Enumerated from 118 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Calgary", @@ -937,7 +1777,17 @@ "iso3166_1": "CA", "latitude": 51.0447, "longitude": -114.0719, - "local_languages": ["en-CA"] + "local_languages": ["en-CA"], + "postal_code": { + "country_format_regex": "^[ABCEGHJ-NPRSTVXY]\\d[ABCEGHJ-NPRSTV-Z] ?\\d[ABCEGHJ-NPRSTV-Z]\\d$", + "city_regex": [ + "^T(?:1Y|2[A-CEGHJ-NPR-TV-Z]|3[A-CEGHJ-NPRS]) ?\\d[ABCEGHJ-NPRSTV-Z]\\d$" + ], + "examples": ["T1Y 1A1", "T2A 1A1", "T2B 1A1"], + "source": "derived from GeoNames postal data (sub, r=30km)", + "codes_matched": 35, + "notes": "Calgary uses T1Y, T2A-T2Z and T3A-T3R; the T1/T3 blocks also include nearby communities. Dataset is fsa-level; local delivery unit appended. Enumerated from 35 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Halifax", @@ -947,7 +1797,15 @@ "iso3166_1": "CA", "latitude": 44.6488, "longitude": -63.5752, - "local_languages": ["en-CA"] + "local_languages": ["en-CA"], + "postal_code": { + "country_format_regex": "^[ABCEGHJ-NPRSTVXY]\\d[ABCEGHJ-NPRSTV-Z] ?\\d[ABCEGHJ-NPRSTV-Z]\\d$", + "city_regex": ["^B3[HJ-NPRS] ?\\d[ABCEGHJ-NPRSTV-Z]\\d$"], + "examples": ["B3H 1A1", "B3J 1A1", "B3K 1A1"], + "source": "derived from GeoNames postal data (sub, r=25km)", + "codes_matched": 9, + "notes": "Halifax peninsula/urban = B3x. The wider municipality also uses B2x and B4x. Dataset is fsa-level; local delivery unit appended. Enumerated from 9 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Mexico City", @@ -957,7 +1815,17 @@ "iso3166_1": "MX", "latitude": 19.4326, "longitude": -99.1332, - "local_languages": ["es-MX"] + "local_languages": ["es-MX"], + "postal_code": { + "country_format_regex": "^\\d{5}$", + "city_regex": [ + "^(?:0(?:1(?:[0247]\\d|1[0-8]|3[1-478]|5[0-689]|6[0-5]|8[0-24-7]|9[049])|2(?:[13][0-6]|[49]\\d|0[0-24-9]|2[03-6]|5[0-4]|6[03-8]|7[0-35-9]|8[013-7])|3(?:[3479][0-4]|0[0-2]|1[09]|2[02-4]|5\\d|6[0-6]|8[0-24])|4(?:0[0-4]|1[0-2]|2[013-8]|3\\d|4[0-24-9]|5[013]|6[0-6]|7[0-3]|8[013-579]|9[0-8])|5(?:0[0236]|1[0-3]|2[0-46-8]|3[1-467]|5[03]|6[02])|6(?:0\\d|1[047]|2[024578]|3[05]|4[0357]|5[089]|6[09]|7[024689]|8[024-9]|9[029])|7(?:[289]\\d|[3-5][0-8]|0[0-24-9]|1[013-9]|6[02-8]|7[02-9])|8(?:[028][0-4]|1[06-8]|3[0-2]|4[02]|5[0-268]|6[0-25]|7[0-367]|9[0-3])|9(?:[278]\\d|[14][0-8]|0[0-46-9]|3[0-356]|5[0-35-7]|6[02-46-9]|9[0-467]))|1(?:0(?:[04]0|1[013]|2[09]|3[0235-8]|5[08]|6[0-5]|7[0-4]|8[1-4]|9[0-2])|1(?:[17]0|[24]\\d|[38][0-7]|0[04]|5[0-689]|6[0145]|9[1-35])|2(?:0[078]|20)|3(?:0[0367]|1[128]|2[013578]|3[016]|4[13]|55|6[04])|4(?:[03]\\d|1[1-6]|2[0-7]|4[0-68]|6[0-5])|5(?:[48][0-7]|0[0-2]|1[02]|2\\d|3[0-57-9]|5[0-4]|6[0-8]|7[0-5]|9[04-9])|6(?:0[1-57]|10|3[0-4]|4[14]|62|7[1-58]|8[168])))\\d$" + ], + "examples": ["01000", "01010", "01020"], + "source": "derived from GeoNames postal data (radius, r=20km)", + "codes_matched": 1207, + "notes": "Mexico City = 01000-16999. Compressed one digit short of full length. Enumerated from 1207 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Tijuana", @@ -967,7 +1835,17 @@ "iso3166_1": "MX", "latitude": 32.5149, "longitude": -117.0382, - "local_languages": ["es-MX"] + "local_languages": ["es-MX"], + "postal_code": { + "country_format_regex": "^\\d{5}$", + "city_regex": [ + "^22(?:0(?:[23][045]|00|4[46])|1(?:15|2[03-7]|6[03-5]|70|94)|2(?:06|1[04-7]|50)|4(?:0[056]|24|4[046]|57)|5(?:0[04-6]|1[05-7]|2[3-6]|3[04-7]|5[057]|6[03-7])|6(?:0[04-6]|1[046]|2[4-6]|4[03-5]|5[04]|6[03-7]))$" + ], + "examples": ["22000", "22020", "22024"], + "source": "derived from GeoNames postal data (radius, r=20km)", + "codes_matched": 85, + "notes": "Tijuana = 22000-22699. Enumerated from 85 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Monterrey", @@ -977,7 +1855,17 @@ "iso3166_1": "MX", "latitude": 25.6866, "longitude": -100.3161, - "local_languages": ["es-MX"] + "local_languages": ["es-MX"], + "postal_code": { + "country_format_regex": "^\\d{5}$", + "city_regex": [ + "^64(?:0(?:[235-7]0|0[04679]|1[089]|4[09])|1(?:[239]0|0[024-9]|1[06-9]|4[058]|5[07-9]|6[05-8]|7[08]|8[06])|2(?:[7-9]0|0[04-689]|1[0589]|2[05-9]|3[03-8]|4[04-9]|5[089]|6[04-69])|3(?:[0-257-9]0|3[09]|4[04-9]|6[05-9])|4(?:[0135-79]0|4[02]|8[089])|5(?:[06][08]|[58]0|1[07]|3[06]|4[09]|7[01])|6(?:[06]0|[45][09]|1[089]|2[03])|7(?:[037]0|15|2[03]|5[034]|6[0459]|8[038]|9[08])|8(?:[0-28]0|3[0346]|4[04-69]|5[089]|6[05]|78|9[037-9])|9(?:[1356]0|[04][09]|2[056]|7[089]|8[034679]|9[067]))$" + ], + "examples": ["64000", "64004", "64006"], + "source": "derived from GeoNames postal data (radius, r=18km)", + "codes_matched": 212, + "notes": "Monterrey city = 64000-64999; the metro area extends into 65xxx-67xxx. Enumerated from 212 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Guadalajara", @@ -987,7 +1875,17 @@ "iso3166_1": "MX", "latitude": 20.6597, "longitude": -103.3496, - "local_languages": ["es-MX"] + "local_languages": ["es-MX"], + "postal_code": { + "country_format_regex": "^\\d{5}$", + "city_regex": [ + "^44(?:009|1(?:[01346-9]0|5[08])|2(?:[0378]0|1[04-9]|2[07]|4[0679]|5[07-9]|6[05-8]|9[089])|3(?:[3-8]0|0[067]|2[09]|9[0589])|4(?:[02457]0|1[09]|3[08]|6[079]|9[03])|5(?:[1268]0|[045][09]|3[058])|6(?:[029]0|[18][09]|3[0589]|4[78]|5[06-9]|6[0578]|7[089])|7(?:[078]0|1[069]|2[03-79]|3[079]|4[08]|5[057]|6[05689]|9[09])|8(?:[046]0|1[0578]|2[0457-9]|7[09]|9[05-9])|9(?:[039]0|[14][089]|2[09]|5[069]|6[0679]|7[078]|8[04-79]))$" + ], + "examples": ["44009", "44100", "44110"], + "source": "derived from GeoNames postal data (radius, r=15km)", + "codes_matched": 183, + "notes": "Guadalajara city = 44100-44999; Zapopan/Tlaquepaque use 45xxx. Enumerated from 183 observed codes; the country_format_regex is the permissive fallback." + } }, { "city": "Ensenada", @@ -997,6 +1895,16 @@ "iso3166_1": "MX", "latitude": 31.8667, "longitude": -116.5964, - "local_languages": ["es-MX"] + "local_languages": ["es-MX"], + "postal_code": { + "country_format_regex": "^\\d{5}$", + "city_regex": [ + "^228(?:[03]0|1[02-589]|2[023]|4[027]|5[0259]|6[03]|7[0239]|8[07-9]|9[578])$" + ], + "examples": ["22800", "22810", "22812"], + "source": "derived from GeoNames postal data (radius, r=15km)", + "codes_matched": 32, + "notes": "Ensenada = 22800-22899. Enumerated from 32 observed codes; the country_format_regex is the permissive fallback." + } } ] diff --git a/tooling/pipeline/src/biergarten_pipeline_orchestrator/biergarten_pipeline_orchestrator.cc b/tooling/pipeline/src/biergarten_pipeline_orchestrator/biergarten_pipeline_orchestrator.cc index eae518aa..59b09ed0 100644 --- a/tooling/pipeline/src/biergarten_pipeline_orchestrator/biergarten_pipeline_orchestrator.cc +++ b/tooling/pipeline/src/biergarten_pipeline_orchestrator/biergarten_pipeline_orchestrator.cc @@ -13,10 +13,12 @@ BiergartenPipelineOrchestrator::BiergartenPipelineOrchestrator( std::unique_ptr generator, std::unique_ptr exporter, std::unique_ptr curated_data_service, + std::unique_ptr postal_code_service, const ApplicationOptions& application_options) : logger_(std::move(logger)), context_service_(std::move(context_service)), generator_(std::move(generator)), exporter_(std::move(exporter)), curated_data_service_(std::move(curated_data_service)), + postal_code_service_(std::move(postal_code_service)), application_options_(application_options) {} \ No newline at end of file diff --git a/tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_breweries.cc b/tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_breweries.cc index 0ee023b4..69bade26 100644 --- a/tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_breweries.cc +++ b/tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_breweries.cc @@ -22,12 +22,15 @@ void BiergartenPipelineOrchestrator::GenerateBreweries( const auto generate_record = [this, &skipped_count]( - const Location& location, - const std::string& region_context) -> std::optional { + const EnrichedCity& enriched_city) -> std::optional { try { - const BreweryResult brewery = - generator_->GenerateBrewery(location, region_context); - return BreweryRecord{.location = location, .brewery = brewery}; + const BreweryResult brewery = generator_->GenerateBrewery(enriched_city); + const std::string postal_code = + postal_code_service_->GeneratePostalCode(enriched_city.location); + return BreweryRecord{ + .address = BreweryAddress{.city = enriched_city.location, + .postal_code = postal_code}, + .brewery = brewery}; } catch (const std::exception& e) { ++skipped_count; @@ -36,7 +39,8 @@ void BiergartenPipelineOrchestrator::GenerateBreweries( .phase = PipelinePhase::BreweryAndBeerGeneration, .message = std::format("[Pipeline] Skipping city '{}' ({}): brewery " "generation failed: {}", - location.city, location.country, e.what())}); + enriched_city.location.city, + enriched_city.location.country, e.what())}); return std::nullopt; } }; @@ -47,19 +51,18 @@ void BiergartenPipelineOrchestrator::GenerateBreweries( exporter_->ProcessRecord(record); } catch (const std::exception& export_exception) { ++export_failed_count; - logger_->Log( - {.level = LogLevel::Warn, - .phase = PipelinePhase::BreweryAndBeerGeneration, - .message = std::format("[Pipeline] Generated brewery for '{}' ({}) " - "but SQLite export failed: {}", - record.location.city, record.location.country, - export_exception.what())}); + logger_->Log({.level = LogLevel::Warn, + .phase = PipelinePhase::BreweryAndBeerGeneration, + .message = std::format( + "[Pipeline] Generated brewery for '{}' ({}) " + "but SQLite export failed: {}", + record.address.city.city, record.address.city.country, + export_exception.what())}); } }; - for (const auto& [location, region_context] : cities) { - const std::optional record = - generate_record(location, region_context); + for (const EnrichedCity& enriched_city : cities) { + const std::optional record = generate_record(enriched_city); if (!record.has_value()) { continue; } diff --git a/tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc b/tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc index bbfe52d1..a16e85a2 100644 --- a/tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc +++ b/tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc @@ -135,9 +135,12 @@ void BiergartenPipelineOrchestrator::GenerateUsers( try { const UserResult user = generator_->GenerateUser(city, persona, sampled_name); + const std::string postal_code = + postal_code_service_->GeneratePostalCode(city.location); return UserRecord{ - .location = city.location, + .address = + UserAddress{.city = city.location, .postal_code = postal_code}, .user = user, .email = BuildEmail(sampled_name, used_email_local_parts), .date_of_birth = GenerateDateOfBirth(rng), @@ -161,13 +164,13 @@ void BiergartenPipelineOrchestrator::GenerateUsers( } catch (const std::exception& export_exception) { ++export_failed_count; - logger_->Log( - {.level = LogLevel::Warn, - .phase = PipelinePhase::UserGeneration, - .message = std::format("[Pipeline] Generated user for '{}' ({}) but " - "SQLite export failed: {}", - record.location.city, record.location.country, - export_exception.what())}); + logger_->Log({.level = LogLevel::Warn, + .phase = PipelinePhase::UserGeneration, + .message = std::format( + "[Pipeline] Generated user for '{}' ({}) but " + "SQLite export failed: {}", + record.address.city.city, record.address.city.country, + export_exception.what())}); } }; diff --git a/tooling/pipeline/src/biergarten_pipeline_orchestrator/log_results.cc b/tooling/pipeline/src/biergarten_pipeline_orchestrator/log_results.cc index 9587ced3..0fa7af00 100644 --- a/tooling/pipeline/src/biergarten_pipeline_orchestrator/log_results.cc +++ b/tooling/pipeline/src/biergarten_pipeline_orchestrator/log_results.cc @@ -14,20 +14,20 @@ void BiergartenPipelineOrchestrator::LogResults() const { boost::json::array brewery_output; - for (const auto& [location, brewery] : generated_breweries_) { + for (const auto& [address, brewery] : generated_breweries_) { + const City& city = address.city; brewery_output.push_back(boost::json::object{ {"name_en", brewery.name_en}, {"description_en", brewery.description_en}, {"name_local", brewery.name_local}, {"description_local", brewery.description_local}, - {"location", boost::json::object{ - {"city", location.city}, - {"country", location.country}, - {"state_province", location.state_province}, - {"iso3166_2", location.iso3166_2}, - {"latitude", location.latitude}, - {"longitude", location.longitude}, - }}}); + {"address", boost::json::object{ + {"city", city.city}, + {"country", city.country}, + {"state_province", city.state_province}, + {"iso3166_2", city.iso3166_2}, + {"postal_code", address.postal_code}, + }}}); } std::ostringstream brewery_oss; diff --git a/tooling/pipeline/src/biergarten_pipeline_orchestrator/query_cities_with_countries.cc b/tooling/pipeline/src/biergarten_pipeline_orchestrator/query_cities_with_countries.cc index f3549dd4..76999f43 100644 --- a/tooling/pipeline/src/biergarten_pipeline_orchestrator/query_cities_with_countries.cc +++ b/tooling/pipeline/src/biergarten_pipeline_orchestrator/query_cities_with_countries.cc @@ -14,13 +14,13 @@ #include "services/curated_data/curated_json_data_service.h" #include "services/logging/logger.h" -std::vector -BiergartenPipelineOrchestrator::QueryCitiesWithCountries() { +std::vector +BiergartenPipelineOrchestrator::QueryLocations() { logger_->Log({.level = LogLevel::Info, .phase = PipelinePhase::Startup, .message = "=== GEOGRAPHIC DATA OVERVIEW ==="}); - const std::vector& all_locations = + const std::vector& all_locations = curated_data_service_->LoadLocations(); const size_t sample_count = std::min( @@ -31,7 +31,7 @@ BiergartenPipelineOrchestrator::QueryCitiesWithCountries() { static_cast>( sample_count); - std::vector sampled_locations; + std::vector sampled_locations; sampled_locations.reserve(sample_count); std::random_device random_generator; diff --git a/tooling/pipeline/src/biergarten_pipeline_orchestrator/run.cc b/tooling/pipeline/src/biergarten_pipeline_orchestrator/run.cc index 5ada0f6f..8e3bb25b 100644 --- a/tooling/pipeline/src/biergarten_pipeline_orchestrator/run.cc +++ b/tooling/pipeline/src/biergarten_pipeline_orchestrator/run.cc @@ -14,7 +14,7 @@ bool BiergartenPipelineOrchestrator::Run() { try { exporter_->Initialize(); - std::vector cities = QueryCitiesWithCountries(); + std::vector cities = QueryLocations(); std::vector enriched; enriched.reserve(cities.size()); diff --git a/tooling/pipeline/src/data_generation/llama/generate_brewery.cc b/tooling/pipeline/src/data_generation/llama/generate_brewery.cc index 30c4ac62..f9362bd2 100644 --- a/tooling/pipeline/src/data_generation/llama/generate_brewery.cc +++ b/tooling/pipeline/src/data_generation/llama/generate_brewery.cc @@ -36,7 +36,9 @@ static std::string FormatLocalLanguageCodes( static constexpr int kBreweryInitialMaxTokens = 2800; BreweryResult LlamaGenerator::GenerateBrewery( - const Location& location, const std::string& region_context) { + const EnrichedCity& enriched_city) { + const City& location = enriched_city.location; + const std::string& region_context = enriched_city.region_context; /** * Preprocess and truncate region context to manageable size */ @@ -66,7 +68,7 @@ BreweryResult LlamaGenerator::GenerateBrewery( * Store location context for retry prompts (without repeating full context) */ const std::string retry_location = - std::format("Location: {}{}\nLocal language codes: {}", location.city, + std::format("City: {}{}\nLocal language codes: {}", location.city, country_suffix, local_language_codes); /** diff --git a/tooling/pipeline/src/data_generation/mock/deterministic_hash.cc b/tooling/pipeline/src/data_generation/mock/deterministic_hash.cc index 06d46b7e..26adc6aa 100644 --- a/tooling/pipeline/src/data_generation/mock/deterministic_hash.cc +++ b/tooling/pipeline/src/data_generation/mock/deterministic_hash.cc @@ -8,14 +8,14 @@ #include "data_generation/mock_generator.h" -size_t MockGenerator::DeterministicHash(const Location& location) { +size_t MockGenerator::DeterministicHash(const City& location) { size_t seed = 0; boost::hash_combine(seed, location.city); boost::hash_combine(seed, location.country); return seed; } -size_t MockGenerator::DeterministicHash(const Location& location, +size_t MockGenerator::DeterministicHash(const City& location, const UserPersona& persona, const Name& name) { size_t seed = DeterministicHash(location); diff --git a/tooling/pipeline/src/data_generation/mock/generate_brewery.cc b/tooling/pipeline/src/data_generation/mock/generate_brewery.cc index 20a4ff6e..ea4d56c7 100644 --- a/tooling/pipeline/src/data_generation/mock/generate_brewery.cc +++ b/tooling/pipeline/src/data_generation/mock/generate_brewery.cc @@ -11,7 +11,8 @@ #include "data_generation/mock_generator.h" BreweryResult MockGenerator::GenerateBrewery( - const Location& location, const std::string& /*region_context*/) { + const EnrichedCity& enriched_city) { + const City& location = enriched_city.location; const size_t hash = DeterministicHash(location); const std::string_view adjective = diff --git a/tooling/pipeline/src/main.cc b/tooling/pipeline/src/main.cc index 3086ff61..b4b590d5 100644 --- a/tooling/pipeline/src/main.cc +++ b/tooling/pipeline/src/main.cc @@ -86,6 +86,24 @@ int main(const int argc, char** argv) { di::bind().to(options), di::bind().to(model_path), di::bind().to(), + di::bind().to( + [options, &log_producer]() -> std::unique_ptr { + if (options.generator.use_mocked) { + log_producer->Log( + {.level = LogLevel::Info, + .phase = PipelinePhase::Startup, + .message = "Postal code: mock (curated examples)"}); + + return std::make_unique(); + } + + log_producer->Log( + {.level = LogLevel::Info, + .phase = PipelinePhase::Startup, + .message = "Postal code: Xeger (regex-based generation)"}); + + return std::make_unique(); + }), di::bind().to( [options, &log_producer]() -> std::unique_ptr { if (options.generator.use_mocked) { @@ -226,4 +244,4 @@ int main(const int argc, char** argv) { return shutdown(EXIT_FAILURE); } -} \ No newline at end of file +} diff --git a/tooling/pipeline/src/services/curated_data/curated_json_data_service.cc b/tooling/pipeline/src/services/curated_data/curated_json_data_service.cc index d9a14c37..cf8b11f3 100644 --- a/tooling/pipeline/src/services/curated_data/curated_json_data_service.cc +++ b/tooling/pipeline/src/services/curated_data/curated_json_data_service.cc @@ -1,6 +1,6 @@ /** * @file json_handling/json_loader.cc - * @brief Parses curated location JSON input into strongly typed Location + * @brief Parses curated location JSON input into strongly typed City * records with strict field validation and descriptive error reporting. */ @@ -30,16 +30,6 @@ static std::string ReadRequiredString(const boost::json::object& object, return std::string(text); } -static double ReadRequiredNumber(const boost::json::object& object, - const char* key) { - const boost::json::value* value = object.if_contains(key); - if (value == nullptr || !value->is_number()) { - throw std::runtime_error( - std::format("Missing or invalid numeric field: {}", key)); - } - return value->to_number(); -} - static std::vector ReadRequiredStringArray( const boost::json::object& object, const char* key) { const boost::json::value* value = object.if_contains(key); @@ -61,6 +51,21 @@ static std::vector ReadRequiredStringArray( return items; } +static PostalCodeSpec ReadPostalCodeSpec(const boost::json::object& object) { + const boost::json::value* value = object.if_contains("postal_code"); + if (value == nullptr || !value->is_object()) { + throw std::runtime_error("Missing or invalid object field: postal_code"); + } + + const auto& postal_code = value->as_object(); + return PostalCodeSpec{ + .country_format_regex = + ReadRequiredString(postal_code, "country_format_regex"), + .city_regexes = ReadRequiredStringArray(postal_code, "city_regex"), + .examples = ReadRequiredStringArray(postal_code, "examples"), + }; +} + namespace { boost::json::value ParseJsonFile(const std::filesystem::path& filepath, @@ -133,15 +138,14 @@ const LocationsList& CuratedJsonDataService::LoadLocations() { } const auto& object = item.as_object(); - locations.push_back(Location{ + locations.push_back(City{ .city = ReadRequiredString(object, "city"), .state_province = ReadRequiredString(object, "state_province"), .iso3166_2 = ReadRequiredString(object, "iso3166_2"), .country = ReadRequiredString(object, "country"), .iso3166_1 = ReadRequiredString(object, "iso3166_1"), .local_languages = ReadRequiredStringArray(object, "local_languages"), - .latitude = ReadRequiredNumber(object, "latitude"), - .longitude = ReadRequiredNumber(object, "longitude"), + .postal_code = ReadPostalCodeSpec(object), }); } cache_.locations = std::move(locations); diff --git a/tooling/pipeline/src/services/curated_data/mock_curated_data_service.cc b/tooling/pipeline/src/services/curated_data/mock_curated_data_service.cc index dda41840..3332fed7 100644 --- a/tooling/pipeline/src/services/curated_data/mock_curated_data_service.cc +++ b/tooling/pipeline/src/services/curated_data/mock_curated_data_service.cc @@ -7,38 +7,46 @@ MockCuratedDataService::MockCuratedDataService() : locations_{ - Location{.city = "Portland", - .state_province = "Oregon", - .iso3166_2 = "US-OR", - .country = "United States", - .iso3166_1 = "US", - .local_languages = {"en"}, - .latitude = 45.5152, - .longitude = -122.6784}, - Location{.city = "Munich", - .state_province = "Bavaria", - .iso3166_2 = "DE-BY", - .country = "Germany", - .iso3166_1 = "DE", - .local_languages = {"de"}, - .latitude = 48.1351, - .longitude = 11.5820}, - Location{.city = "Lyon", - .state_province = "Auvergne-Rhone-Alpes", - .iso3166_2 = "FR-ARA", - .country = "France", - .iso3166_1 = "FR", - .local_languages = {"fr"}, - .latitude = 45.7640, - .longitude = 4.8357}, - Location{.city = "Brussels", - .state_province = "Brussels-Capital", - .iso3166_2 = "BE-BRU", - .country = "Belgium", - .iso3166_1 = "BE", - .local_languages = {"nl", "fr"}, - .latitude = 50.8503, - .longitude = 4.3517}, + City{.city = "Portland", + .state_province = "Oregon", + .iso3166_2 = "US-OR", + .country = "United States", + .iso3166_1 = "US", + .local_languages = {"en"}, + .postal_code = + PostalCodeSpec{.country_format_regex = R"(^\d{5}$)", + .city_regexes = {R"(^972\d{2}$)"}, + .examples = {"97201", "97202", "97209"}}}, + City{.city = "Munich", + .state_province = "Bavaria", + .iso3166_2 = "DE-BY", + .country = "Germany", + .iso3166_1 = "DE", + .local_languages = {"de"}, + .postal_code = + PostalCodeSpec{.country_format_regex = R"(^\d{5}$)", + .city_regexes = {R"(^8\d{4}$)"}, + .examples = {"80331", "80333", "81667"}}}, + City{.city = "Lyon", + .state_province = "Auvergne-Rhone-Alpes", + .iso3166_2 = "FR-ARA", + .country = "France", + .iso3166_1 = "FR", + .local_languages = {"fr"}, + .postal_code = + PostalCodeSpec{.country_format_regex = R"(^\d{5}$)", + .city_regexes = {R"(^690\d{2}$)"}, + .examples = {"69001", "69002", "69007"}}}, + City{.city = "Brussels", + .state_province = "Brussels-Capital", + .iso3166_2 = "BE-BRU", + .country = "Belgium", + .iso3166_1 = "BE", + .local_languages = {"nl", "fr"}, + .postal_code = + PostalCodeSpec{.country_format_regex = R"(^\d{4}$)", + .city_regexes = {R"(^1\d{3}$)"}, + .examples = {"1000", "1050", "1180"}}}, }, personas_{ UserPersona{.name = "Hophead Explorer", diff --git a/tooling/pipeline/src/services/enrichment/wikipedia/get_summary.cc b/tooling/pipeline/src/services/enrichment/wikipedia/get_summary.cc index 342f84a5..07cfde7d 100644 --- a/tooling/pipeline/src/services/enrichment/wikipedia/get_summary.cc +++ b/tooling/pipeline/src/services/enrichment/wikipedia/get_summary.cc @@ -11,7 +11,7 @@ #include "services/enrichment/wikipedia_service.h" std::string WikipediaEnrichmentService::GetLocationContext( - const Location& loc) { + const City& loc) { using namespace std::literals::chrono_literals; if (!this->client_) { if (logger_) { diff --git a/tooling/pipeline/src/services/postal_code/xeger_postal_code_service.cc b/tooling/pipeline/src/services/postal_code/xeger_postal_code_service.cc new file mode 100644 index 00000000..796c9ab8 --- /dev/null +++ b/tooling/pipeline/src/services/postal_code/xeger_postal_code_service.cc @@ -0,0 +1,44 @@ +/** + * @file services/postal_code/xeger_postal_code_service.cc + * @brief XegerPostalCodeService::GeneratePostalCode() implementation. + */ + +#include "services/postal_code/xeger_postal_code_service.h" + +#include +#include +#include +#include +#include +#include + +#include "services/xeger/xeger.h" + +XegerPostalCodeService::XegerPostalCodeService() + : rng_(std::random_device{}()) {} + +std::string XegerPostalCodeService::GeneratePostalCode(const City& city) { + const PostalCodeSpec& spec = city.postal_code; + + std::vector city_patterns; + for (const std::string& pattern : spec.city_regexes) { + if (!pattern.empty()) { + city_patterns.push_back(pattern); + } + } + + std::string_view pattern; + if (!city_patterns.empty()) { + std::uniform_int_distribution dist(0, city_patterns.size() - 1); + pattern = city_patterns[dist(rng_)]; + } else if (!spec.country_format_regex.empty()) { + pattern = spec.country_format_regex; + } else { + throw std::runtime_error(std::format( + "XegerPostalCodeService: no postal-code pattern available for " + "city '{}'", + city.city)); + } + + return GenerateStringFromRegex(pattern, rng_); +} diff --git a/tooling/pipeline/src/services/prompt_directory.cc b/tooling/pipeline/src/services/prompt_directory.cc index b2ad3703..d567454b 100644 --- a/tooling/pipeline/src/services/prompt_directory.cc +++ b/tooling/pipeline/src/services/prompt_directory.cc @@ -15,9 +15,7 @@ #include #include -// --------------------------------------------------------------------------- // PromptDirectory -// --------------------------------------------------------------------------- PromptDirectory::PromptDirectory(const std::filesystem::path& prompt_dir) : PromptDirectory(prompt_dir, nullptr) {} @@ -27,21 +25,21 @@ PromptDirectory::PromptDirectory(const std::filesystem::path& prompt_dir, : prompt_dir_(prompt_dir), logger_(std::move(logger)) { std::error_code ec; - // Scenario 4: directory must exist. + // Directory must exist. if (!std::filesystem::exists(prompt_dir_, ec) || ec) { throw std::runtime_error( "PromptDirectory: prompt directory does not exist: " + prompt_dir_.string()); } - // Scenario 4: path must be a directory, not a file. + // Path must be a directory, not a file. if (!std::filesystem::is_directory(prompt_dir_, ec) || ec) { throw std::runtime_error( "PromptDirectory: prompt directory path is not a directory: " + prompt_dir_.string()); } - // Scenario 4: directory must be readable (probe with directory_iterator). + // Directory must be readable (probe with directory_iterator). std::filesystem::directory_iterator probe(prompt_dir_, ec); if (ec) { throw std::runtime_error(std::format( @@ -68,7 +66,7 @@ std::string PromptDirectory::Load(std::string_view key) { return cache_it->second; } - // Scenario 3: resolve /.md and require it to exist. + // Resolve /.md and require it to exist. const std::filesystem::path file_path = prompt_dir_ / std::filesystem::path(std::format("{}.md", key_str)); @@ -98,6 +96,7 @@ std::string PromptDirectory::Load(std::string_view key) { key_str, file_path.string(), content.size())}); } + // Cache the loaded content for future calls. cache_.emplace(key_str, content); return content; } diff --git a/tooling/pipeline/src/services/sqlite/finalize.cc b/tooling/pipeline/src/services/sqlite/finalize.cc index c4e4ff64..b8e64574 100644 --- a/tooling/pipeline/src/services/sqlite/finalize.cc +++ b/tooling/pipeline/src/services/sqlite/finalize.cc @@ -14,9 +14,11 @@ void SqliteExportService::Finalize() { } try { + insert_user_address_stmt_.reset(); insert_user_stmt_.reset(); + insert_brewery_address_stmt_.reset(); insert_brewery_stmt_.reset(); - insert_location_stmt_.reset(); + insert_city_stmt_.reset(); if (transaction_open_) { sqlite_export_service_internal::ExecSql( db_handle_, "COMMIT;", "Failed to commit SQLite transaction"); @@ -24,7 +26,7 @@ void SqliteExportService::Finalize() { } db_handle_.reset(); - location_cache_.clear(); + city_cache_.clear(); } catch (...) { RollbackAndCloseNoThrow(); throw; diff --git a/tooling/pipeline/src/services/sqlite/helpers/sqlite_statement_helpers.cc b/tooling/pipeline/src/services/sqlite/helpers/sqlite_statement_helpers.cc index 8b2f75f3..2420ea81 100644 --- a/tooling/pipeline/src/services/sqlite/helpers/sqlite_statement_helpers.cc +++ b/tooling/pipeline/src/services/sqlite/helpers/sqlite_statement_helpers.cc @@ -88,7 +88,8 @@ sqlite3_int64 LastInsertRowId(const SqliteDatabaseHandle& db_handle) { } std::string SerializeVector(const std::vector& str_vec) { - boost::json::array array(str_vec.size()); + boost::json::array array; + array.reserve(str_vec.size()); for (const auto& s : str_vec) { array.emplace_back(s); } diff --git a/tooling/pipeline/src/services/sqlite/initialize.cc b/tooling/pipeline/src/services/sqlite/initialize.cc index 4c7def22..3089ad0c 100644 --- a/tooling/pipeline/src/services/sqlite/initialize.cc +++ b/tooling/pipeline/src/services/sqlite/initialize.cc @@ -10,7 +10,6 @@ #include #include "services/database/sqlite_export_service.h" -#include "services/database/sqlite_export_service_helpers.h" std::filesystem::path SqliteExportService::BuildDatabasePath() const { std::filesystem::path base_filename("biergarten_seed_" + run_timestamp_utc_ + @@ -28,26 +27,40 @@ std::filesystem::path SqliteExportService::BuildDatabasePath() const { void SqliteExportService::InitializeSchema() const { sqlite_export_service_internal::ExecSql( - db_handle_, sqlite_export_service_internal::kCreateLocationsTableSql, - "Failed to create SQLite locations table"); + db_handle_, sqlite_export_service_internal::kCreateCitiesTableSql, + "Failed to create SQLite cities table"); sqlite_export_service_internal::ExecSql( db_handle_, sqlite_export_service_internal::kCreateBreweriesTableSql, "Failed to create SQLite breweries table"); + sqlite_export_service_internal::ExecSql( + db_handle_, + sqlite_export_service_internal::kCreateBreweryAddressesTableSql, + "Failed to create SQLite brewery_addresses table"); sqlite_export_service_internal::ExecSql( db_handle_, sqlite_export_service_internal::kCreateUsersTableSql, "Failed to create SQLite users table"); + sqlite_export_service_internal::ExecSql( + db_handle_, sqlite_export_service_internal::kCreateUserAddressesTableSql, + "Failed to create SQLite user_addresses table"); } void SqliteExportService::PrepareStatements() { - insert_location_stmt_ = sqlite_export_service_internal::PrepareStatement( - db_handle_, sqlite_export_service_internal::kInsertLocationSql, - "Failed to prepare SQLite location insert statement"); + insert_city_stmt_ = sqlite_export_service_internal::PrepareStatement( + db_handle_, sqlite_export_service_internal::kInsertCitySql, + "Failed to prepare SQLite city insert statement"); insert_brewery_stmt_ = sqlite_export_service_internal::PrepareStatement( db_handle_, sqlite_export_service_internal::kInsertBrewerySql, "Failed to prepare SQLite brewery insert statement"); + insert_brewery_address_stmt_ = + sqlite_export_service_internal::PrepareStatement( + db_handle_, sqlite_export_service_internal::kInsertBreweryAddressSql, + "Failed to prepare SQLite brewery address insert statement"); insert_user_stmt_ = sqlite_export_service_internal::PrepareStatement( db_handle_, sqlite_export_service_internal::kInsertUserSql, "Failed to prepare SQLite user insert statement"); + insert_user_address_stmt_ = sqlite_export_service_internal::PrepareStatement( + db_handle_, sqlite_export_service_internal::kInsertUserAddressSql, + "Failed to prepare SQLite user address insert statement"); } void SqliteExportService::RollbackAndCloseNoThrow() noexcept { @@ -60,11 +73,13 @@ void SqliteExportService::RollbackAndCloseNoThrow() noexcept { transaction_open_ = false; } + insert_user_address_stmt_.reset(); insert_user_stmt_.reset(); + insert_brewery_address_stmt_.reset(); insert_brewery_stmt_.reset(); - insert_location_stmt_.reset(); + insert_city_stmt_.reset(); db_handle_.reset(); - location_cache_.clear(); + city_cache_.clear(); } void SqliteExportService::Initialize() { diff --git a/tooling/pipeline/src/services/sqlite/process_record.cc b/tooling/pipeline/src/services/sqlite/process_record.cc index 9acc0fc3..a4cf9f1a 100644 --- a/tooling/pipeline/src/services/sqlite/process_record.cc +++ b/tooling/pipeline/src/services/sqlite/process_record.cc @@ -1,10 +1,9 @@ /** * @file services/sqlite/process_record.cc * @brief SqliteExportService::ProcessRecord() implementation - * and the shared location-resolution helper. + * and the shared city-resolution helper. */ -#include #include #include #include @@ -12,94 +11,91 @@ #include "services/database/sqlite_export_service.h" #include "services/database/sqlite_export_service_helpers.h" -constexpr int kLocationPrecision = 17; - -std::string SqliteExportService::BuildLocationKey(const Location& location) { +std::string SqliteExportService::BuildCityKey(const City& city) { std::ostringstream key_stream; - key_stream << location.city << '\n' - << location.state_province << '\n' - << location.iso3166_2 << '\n' - << location.country << '\n' - << location.iso3166_1 << '\n' - << std::setprecision(kLocationPrecision) << location.latitude - << '\n' - << std::setprecision(kLocationPrecision) << location.longitude - << '\n' + key_stream << city.city << '\n' + << city.state_province << '\n' + << city.iso3166_2 << '\n' + << city.country << '\n' + << city.iso3166_1 << '\n' << sqlite_export_service_internal::SerializeVector( - location.local_languages); + city.local_languages); return key_stream.str(); } -sqlite3_int64 SqliteExportService::ResolveLocationId(const Location& location) { - const std::string location_key = BuildLocationKey(location); - const auto cached_location = location_cache_.find(location_key); - if (cached_location != location_cache_.end()) { - return cached_location->second; +sqlite3_int64 SqliteExportService::ResolveCityId(const City& city) { + const std::string city_key = BuildCityKey(city); + const auto cached_city = city_cache_.find(city_key); + if (cached_city != city_cache_.end()) { + return cached_city->second; } const std::string local_languages_json = - sqlite_export_service_internal::SerializeVector(location.local_languages); + sqlite_export_service_internal::SerializeVector(city.local_languages); + const std::string city_regex_json = + sqlite_export_service_internal::SerializeVector( + city.postal_code.city_regexes); sqlite_export_service_internal::Bind( - insert_location_stmt_, + insert_city_stmt_, sqlite_export_service_internal::BoundParam{ - .index = sqlite_export_service_internal::kLocationCityBindIndex, - .value = location.city, - .action = "Failed to bind SQLite location city"}); + .index = sqlite_export_service_internal::kCityCityBindIndex, + .value = city.city, + .action = "Failed to bind SQLite city name"}); sqlite_export_service_internal::Bind( - insert_location_stmt_, + insert_city_stmt_, sqlite_export_service_internal::BoundParam{ - .index = - sqlite_export_service_internal::kLocationStateProvinceBindIndex, - .value = location.state_province, - .action = "Failed to bind SQLite location state/province"}); + .index = sqlite_export_service_internal::kCityStateProvinceBindIndex, + .value = city.state_province, + .action = "Failed to bind SQLite city state/province"}); sqlite_export_service_internal::Bind( - insert_location_stmt_, + insert_city_stmt_, sqlite_export_service_internal::BoundParam{ - .index = sqlite_export_service_internal::kLocationIso31662BindIndex, - .value = location.iso3166_2, - .action = "Failed to bind SQLite location ISO 3166-2 code"}); + .index = sqlite_export_service_internal::kCityIso31662BindIndex, + .value = city.iso3166_2, + .action = "Failed to bind SQLite city ISO 3166-2 code"}); sqlite_export_service_internal::Bind( - insert_location_stmt_, + insert_city_stmt_, sqlite_export_service_internal::BoundParam{ - .index = sqlite_export_service_internal::kLocationCountryBindIndex, - .value = location.country, - .action = "Failed to bind SQLite location country"}); + .index = sqlite_export_service_internal::kCityCountryBindIndex, + .value = city.country, + .action = "Failed to bind SQLite city country"}); sqlite_export_service_internal::Bind( - insert_location_stmt_, + insert_city_stmt_, sqlite_export_service_internal::BoundParam{ - .index = sqlite_export_service_internal::kLocationIso31661BindIndex, - .value = location.iso3166_1, - .action = "Failed to bind SQLite location ISO 3166-1 code"}); + .index = sqlite_export_service_internal::kCityIso31661BindIndex, + .value = city.iso3166_1, + .action = "Failed to bind SQLite city ISO 3166-1 code"}); sqlite_export_service_internal::Bind( - insert_location_stmt_, + insert_city_stmt_, sqlite_export_service_internal::BoundParam{ - .index = sqlite_export_service_internal::kLocationLanguagesBindIndex, + .index = sqlite_export_service_internal::kCityLanguagesBindIndex, .value = local_languages_json, - .action = "Failed to bind SQLite location languages"}); + .action = "Failed to bind SQLite city languages"}); sqlite_export_service_internal::Bind( - insert_location_stmt_, - sqlite_export_service_internal::BoundParam{ - .index = sqlite_export_service_internal::kLocationLatitudeBindIndex, - .value = location.latitude, - .action = "Failed to bind SQLite location latitude"}); + insert_city_stmt_, + sqlite_export_service_internal::BoundParam{ + .index = sqlite_export_service_internal:: + kCityPostalCodeCountryFormatRegexBindIndex, + .value = city.postal_code.country_format_regex, + .action = "Failed to bind SQLite city postal-code format regex"}); sqlite_export_service_internal::Bind( - insert_location_stmt_, - sqlite_export_service_internal::BoundParam{ - .index = sqlite_export_service_internal::kLocationLongitudeBindIndex, - .value = location.longitude, - .action = "Failed to bind SQLite location longitude"}); + insert_city_stmt_, + sqlite_export_service_internal::BoundParam{ + .index = sqlite_export_service_internal:: + kCityPostalCodeCityRegexJsonBindIndex, + .value = city_regex_json, + .action = "Failed to bind SQLite city postal-code city regexes"}); sqlite_export_service_internal::StepStatement( - db_handle_, insert_location_stmt_, - "Failed to insert SQLite location row"); + db_handle_, insert_city_stmt_, "Failed to insert SQLite city row"); - const sqlite3_int64 location_id = + const sqlite3_int64 city_id = sqlite_export_service_internal::LastInsertRowId(db_handle_); - location_cache_.emplace(location_key, location_id); - sqlite_export_service_internal::ResetStatement(insert_location_stmt_); + city_cache_.emplace(city_key, city_id); + sqlite_export_service_internal::ResetStatement(insert_city_stmt_); - return location_id; + return city_id; } uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) { @@ -107,14 +103,7 @@ uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) { throw std::runtime_error("SQLite export service is not initialized"); } - const sqlite3_int64 location_id = ResolveLocationId(brewery.location); - - sqlite_export_service_internal::Bind( - insert_brewery_stmt_, - sqlite_export_service_internal::BoundParam{ - .index = sqlite_export_service_internal::kBreweryLocationIdBindIndex, - .value = location_id, - .action = "Failed to bind SQLite brewery location id"}); + const sqlite3_int64 city_id = ResolveCityId(brewery.address.city); sqlite_export_service_internal::Bind( insert_brewery_stmt_, @@ -149,7 +138,38 @@ uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) { sqlite_export_service_internal::StepStatement( db_handle_, insert_brewery_stmt_, "Failed to insert SQLite brewery row"); + const sqlite3_int64 brewery_id = + sqlite_export_service_internal::LastInsertRowId(db_handle_); + sqlite_export_service_internal::ResetStatement(insert_brewery_stmt_); - return sqlite_export_service_internal::LastInsertRowId(db_handle_); + sqlite_export_service_internal::Bind( + insert_brewery_address_stmt_, + sqlite_export_service_internal::BoundParam{ + .index = + sqlite_export_service_internal::kBreweryAddressBreweryIdBindIndex, + .value = brewery_id, + .action = "Failed to bind SQLite brewery address brewery id"}); + sqlite_export_service_internal::Bind( + insert_brewery_address_stmt_, + sqlite_export_service_internal::BoundParam{ + .index = + sqlite_export_service_internal::kBreweryAddressCityIdBindIndex, + .value = city_id, + .action = "Failed to bind SQLite brewery address city id"}); + sqlite_export_service_internal::Bind( + insert_brewery_address_stmt_, + sqlite_export_service_internal::BoundParam{ + .index = sqlite_export_service_internal:: + kBreweryAddressPostalCodeBindIndex, + .value = brewery.address.postal_code, + .action = "Failed to bind SQLite brewery address postal code"}); + + sqlite_export_service_internal::StepStatement( + db_handle_, insert_brewery_address_stmt_, + "Failed to insert SQLite brewery address row"); + + sqlite_export_service_internal::ResetStatement(insert_brewery_address_stmt_); + + return static_cast(brewery_id); } diff --git a/tooling/pipeline/src/services/sqlite/process_user_record.cc b/tooling/pipeline/src/services/sqlite/process_user_record.cc index 34bac070..3b3f369d 100644 --- a/tooling/pipeline/src/services/sqlite/process_user_record.cc +++ b/tooling/pipeline/src/services/sqlite/process_user_record.cc @@ -13,14 +13,8 @@ uint64_t SqliteExportService::ProcessRecord(const UserRecord& user) { throw std::runtime_error("SQLite export service is not initialized"); } - const sqlite3_int64 location_id = ResolveLocationId(user.location); + const sqlite3_int64 city_id = ResolveCityId(user.address.city); - sqlite_export_service_internal::Bind( - insert_user_stmt_, - sqlite_export_service_internal::BoundParam{ - .index = sqlite_export_service_internal::kUserLocationIdBindIndex, - .value = location_id, - .action = "Failed to bind SQLite user location id"}); sqlite_export_service_internal::Bind( insert_user_stmt_, sqlite_export_service_internal::BoundParam{ @@ -73,7 +67,36 @@ uint64_t SqliteExportService::ProcessRecord(const UserRecord& user) { sqlite_export_service_internal::StepStatement( db_handle_, insert_user_stmt_, "Failed to insert SQLite user row"); + const sqlite3_int64 user_id = + sqlite_export_service_internal::LastInsertRowId(db_handle_); + sqlite_export_service_internal::ResetStatement(insert_user_stmt_); - return sqlite_export_service_internal::LastInsertRowId(db_handle_); + sqlite_export_service_internal::Bind( + insert_user_address_stmt_, + sqlite_export_service_internal::BoundParam{ + .index = sqlite_export_service_internal::kUserAddressUserIdBindIndex, + .value = user_id, + .action = "Failed to bind SQLite user address user id"}); + sqlite_export_service_internal::Bind( + insert_user_address_stmt_, + sqlite_export_service_internal::BoundParam{ + .index = sqlite_export_service_internal::kUserAddressCityIdBindIndex, + .value = city_id, + .action = "Failed to bind SQLite user address city id"}); + sqlite_export_service_internal::Bind( + insert_user_address_stmt_, + sqlite_export_service_internal::BoundParam{ + .index = + sqlite_export_service_internal::kUserAddressPostalCodeBindIndex, + .value = user.address.postal_code, + .action = "Failed to bind SQLite user address postal code"}); + + sqlite_export_service_internal::StepStatement( + db_handle_, insert_user_address_stmt_, + "Failed to insert SQLite user address row"); + + sqlite_export_service_internal::ResetStatement(insert_user_address_stmt_); + + return static_cast(user_id); } diff --git a/tooling/pipeline/src/services/xeger/xeger.cc b/tooling/pipeline/src/services/xeger/xeger.cc new file mode 100644 index 00000000..b04ebefb --- /dev/null +++ b/tooling/pipeline/src/services/xeger/xeger.cc @@ -0,0 +1,23 @@ +/** + * @file services/xeger/xeger.cc + * @brief Public entry point: parses a regex pattern into an AST and walks it + * to emit a random matching string ("xeger" -- "regex" backwards). + * + * See xeger_internal.h for the parser/AST/generator this delegates to. + */ + +#include "services/xeger/xeger.h" + +#include +#include +#include + +#include "services/xeger/xeger_internal.h" + +std::string GenerateStringFromRegex(std::string_view pattern, std::mt19937& rng, + const int unbounded_repeat_cap) { + xeger_internal::Parser parser(pattern); + const xeger_internal::NodePtr ast = parser.Parse(); + xeger_internal::XegerGenerator generator(rng, unbounded_repeat_cap); + return generator.Run(*ast); +} diff --git a/tooling/pipeline/src/services/xeger/xeger_generator.cc b/tooling/pipeline/src/services/xeger/xeger_generator.cc new file mode 100644 index 00000000..2fdfb578 --- /dev/null +++ b/tooling/pipeline/src/services/xeger/xeger_generator.cc @@ -0,0 +1,110 @@ +/** + * @file services/xeger/xeger_generator.cc + * @brief Visitor that walks a regex AST and emits a random matching string. + */ + +#include "services/xeger/xeger_internal.h" + +#include +#include +#include +#include +#include + +namespace xeger_internal { + +XegerGenerator::XegerGenerator(std::mt19937& rng, const int unbounded_cap) + : rng_(rng), unbounded_cap_(unbounded_cap) {} + +std::string XegerGenerator::Run(const Node& root) { + out_.clear(); + root.Accept(*this); + return out_; +} + +void XegerGenerator::Visit(const Literal& node) { out_ += node.ch; } + +void XegerGenerator::Visit(const AnyChar&) { + static constexpr int kPrintableLow = ' '; // empty space - character 32 + static constexpr int kPrintableHigh = '~'; // tilde - character 126 + + std::uniform_int_distribution dist(kPrintableLow, kPrintableHigh); + + // assert that the rand number is in the char range + bool inCharRange = dist.min() >= std::numeric_limits::min() && + dist.max() <= std::numeric_limits::max(); + assert(inCharRange && "Random number is out of char range"); + + out_ += static_cast(dist(rng_)); +} + +void XegerGenerator::Visit(const CharClass& node) { + out_ += PickFromClass(node); +} + +void XegerGenerator::Visit(const Concat& node) { + for (const NodePtr& part : node.parts) { + part->Accept(*this); + } +} + +void XegerGenerator::Visit(const Alternation& node) { + std::uniform_int_distribution dist(0, node.options.size() - 1); + node.options[dist(rng_)]->Accept(*this); +} + +void XegerGenerator::Visit(const Repeat& node) { + const int lo = node.min; + const int hi = (node.max < 0) ? node.min + unbounded_cap_ : node.max; + std::uniform_int_distribution dist(lo, hi); + const int count = dist(rng_); + for (int i = 0; i < count; ++i) { + node.child->Accept(*this); + } +} + +bool XegerGenerator::InRanges(const CharClass& node, const char c) { + for (const auto& [lo, hi] : node.ranges) { + if (c >= lo && c <= hi) { + return true; + } + } + return false; +} + +char XegerGenerator::PickFromClass(const CharClass& node) { + if (!node.negated) { + int total = 0; + for (const auto& [lo, hi] : node.ranges) { + total += hi - lo + 1; + } + std::uniform_int_distribution dist(0, total - 1); + int k = dist(rng_); + for (const auto& [lo, hi] : node.ranges) { + const int width = hi - lo + 1; + if (k < width) { + return static_cast(lo + k); + } + k -= width; + } + return node.ranges.back().second; // unreachable + } + + // Negated class: draw random printable characters until one falls + // outside the forbidden set. + static constexpr int kPrintableLow = 32; + static constexpr int kPrintableHigh = 126; + static constexpr int kMaxAttempts = 1000; + std::uniform_int_distribution dist(kPrintableLow, kPrintableHigh); + for (int attempt = 0; attempt < kMaxAttempts; ++attempt) { + const char c = static_cast(dist(rng_)); + if (!InRanges(node, c)) { + return c; + } + } + throw std::runtime_error( + "GenerateStringFromRegex: negated class rules out every printable " + "character"); +} + +} // namespace xeger_internal diff --git a/tooling/pipeline/src/services/xeger/xeger_parser.cc b/tooling/pipeline/src/services/xeger/xeger_parser.cc new file mode 100644 index 00000000..180b6f04 --- /dev/null +++ b/tooling/pipeline/src/services/xeger/xeger_parser.cc @@ -0,0 +1,307 @@ +/** + * @file services/xeger/xeger_parser.cc + * @brief Recursive-descent parser that turns a regex pattern into an AST. + */ + +#include "services/xeger/xeger_internal.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace xeger_internal { + +Parser::Parser(std::string_view pattern) : src_(StripAnchors(pattern)) {} + +NodePtr Parser::Parse() { + NodePtr node = ParseAlternation(); + if (!AtEnd()) { + throw std::runtime_error( + std::format("GenerateStringFromRegex: unexpected '{}' at position {}", + Peek(), pos_)); + } + return node; +} + +std::string Parser::StripAnchors(std::string_view pattern) { + if (!pattern.empty() && pattern.front() == kAnchorStart) { + pattern.remove_prefix(1); + } + const bool has_trailing_dollar = + !pattern.empty() && pattern.back() == kAnchorEnd && + (pattern.size() < 2 || pattern[pattern.size() - 2] != kEscape); + if (has_trailing_dollar) { + pattern.remove_suffix(1); + } + return std::string(pattern); +} + +bool Parser::AtEnd() const { return pos_ >= src_.size(); } +char Parser::Peek() const { return src_[pos_]; } +char Parser::Advance() { return src_[pos_++]; } + +bool Parser::Match(const char c) { + if (!AtEnd() && Peek() == c) { + ++pos_; + return true; + } + return false; +} + +void Parser::Expect(const char c) { + if (!Match(c)) { + throw std::runtime_error(std::format( + "GenerateStringFromRegex: expected '{}' at position {}", c, pos_)); + } +} + +NodePtr Parser::ParseAlternation() { + std::vector options; + options.push_back(ParseConcat()); + while (Match(kAlternation)) { + options.push_back(ParseConcat()); + } + if (options.size() == 1) { + return std::move(options.front()); + } + auto alternation = std::make_unique(); + alternation->options = std::move(options); + return alternation; +} + +NodePtr Parser::ParseConcat() { + auto concat = std::make_unique(); + while (!AtEnd() && Peek() != kAlternation && Peek() != kGroupClose) { + concat->parts.push_back(ParseRepeat()); + } + if (concat->parts.size() == 1) { + return std::move(concat->parts.front()); + } + return concat; +} + +NodePtr Parser::ParseRepeat() { + NodePtr atom = ParseAtom(); + if (AtEnd()) { + return atom; + } + + int lo = 0; + int hi = -1; + switch (Peek()) { + case kStar: + Advance(); + lo = 0; + hi = -1; + break; + case kPlus: + Advance(); + lo = 1; + hi = -1; + break; + case kOptional: + Advance(); + lo = 0; + hi = 1; + break; + case kBraceOpen: + return ParseBrace(std::move(atom)); + default: + return atom; + } + auto repeat = std::make_unique(); + repeat->child = std::move(atom); + repeat->min = lo; + repeat->max = hi; + return repeat; +} + +NodePtr Parser::ParseBrace(NodePtr atom) { + Expect(kBraceOpen); + std::string lower_digits; + std::string upper_digits; + bool has_comma = false; + while (!AtEnd() && std::isdigit(static_cast(Peek())) != 0) { + lower_digits += Advance(); + } + if (Match(kBraceComma)) { + has_comma = true; + while (!AtEnd() && + std::isdigit(static_cast(Peek())) != 0) { + upper_digits += Advance(); + } + } + Expect(kBraceClose); + + auto repeat = std::make_unique(); + repeat->child = std::move(atom); + repeat->min = lower_digits.empty() ? 0 : std::stoi(lower_digits); + if (!has_comma) { + repeat->max = repeat->min; + } else if (upper_digits.empty()) { + repeat->max = -1; + } else { + repeat->max = std::stoi(upper_digits); + } + if (repeat->max != -1 && repeat->max < repeat->min) { + throw std::runtime_error( + "GenerateStringFromRegex: repetition max is smaller than min"); + } + return repeat; +} + +NodePtr Parser::ParseAtom() { + if (AtEnd()) { + throw std::runtime_error( + "GenerateStringFromRegex: pattern ended unexpectedly"); + } + const char c = Peek(); + switch (c) { + case kGroupOpen: { + Advance(); + // Non-capturing group "(?:...)" -- capture semantics don't apply + // here (nothing back-references a group), so it's treated exactly + // like a plain group once the "?:" marker is consumed. + if (!AtEnd() && Peek() == kNonCapturingMarker) { + if (pos_ + 1 < src_.size() && src_[pos_ + 1] == kNonCapturingColon) { + pos_ += 2; + } else { + throw std::runtime_error(std::format( + "GenerateStringFromRegex: unsupported group syntax at " + "position {} (only plain '(...)' and non-capturing " + "'(?:...)' groups are supported)", + pos_)); + } + } + NodePtr inner = ParseAlternation(); + Expect(kGroupClose); + return inner; + } + case kClassOpen: + return ParseCharClass(); + case kAnyChar: + Advance(); + return std::make_unique(); + case kEscape: + Advance(); + return ParseEscape(); + case kStar: + case kPlus: + case kOptional: + case kBraceOpen: + throw std::runtime_error(std::format( + "GenerateStringFromRegex: quantifier with no preceding item at " + "position {}", + pos_)); + default: + Advance(); + return std::make_unique(c); + } +} + +NodePtr Parser::MakeClass(std::vector> ranges, + const bool negated) { + auto char_class = std::make_unique(); + char_class->ranges = std::move(ranges); + char_class->negated = negated; + return char_class; +} + +NodePtr Parser::ParseEscape() { + if (AtEnd()) { + throw std::runtime_error( + "GenerateStringFromRegex: a backslash at the very end has nothing " + "to escape"); + } + const char c = Advance(); + switch (c) { + case 'd': + return MakeClass({{'0', '9'}}, false); + case 'D': + return MakeClass({{'0', '9'}}, true); + case 'w': + return MakeClass({{'a', 'z'}, {'A', 'Z'}, {'0', '9'}, {'_', '_'}}, + false); + case 'W': + return MakeClass({{'a', 'z'}, {'A', 'Z'}, {'0', '9'}, {'_', '_'}}, + true); + case 's': + return MakeClass({{' ', ' '}, {'\t', '\t'}, {'\n', '\n'}, {'\r', '\r'}}, + false); + case 'S': + return MakeClass({{' ', ' '}, {'\t', '\t'}, {'\n', '\n'}, {'\r', '\r'}}, + true); + case 'n': + return std::make_unique('\n'); + case 't': + return std::make_unique('\t'); + case 'r': + return std::make_unique('\r'); + default: + // \. \* \( \\ etc. -- the literal character itself. + return std::make_unique(c); + } +} + +NodePtr Parser::ParseCharClass() { + Expect(kClassOpen); + auto char_class = std::make_unique(); + if (Match(kClassNegate)) { + char_class->negated = true; + } + if (AtEnd()) { + throw std::runtime_error( + "GenerateStringFromRegex: character class '[' is never closed"); + } + + while (!AtEnd() && Peek() != kClassClose) { + char lo = ClassChar(); + const bool is_range = Peek() == kClassRange && pos_ + 1 < src_.size() && + src_[pos_ + 1] != kClassClose; + if (is_range) { + Advance(); + char hi = ClassChar(); + if (hi < lo) { + std::swap(lo, hi); + } + char_class->ranges.emplace_back(lo, hi); + } else { + char_class->ranges.emplace_back(lo, lo); + } + } + Expect(kClassClose); + if (char_class->ranges.empty()) { + throw std::runtime_error( + "GenerateStringFromRegex: empty character class []"); + } + return char_class; +} + +char Parser::ClassChar() { + const char c = Advance(); + if (c != kEscape) { + return c; + } + if (AtEnd()) { + throw std::runtime_error( + "GenerateStringFromRegex: backslash at end of character class"); + } + const char escaped = Advance(); + switch (escaped) { + case 'n': + return '\n'; + case 't': + return '\t'; + case 'r': + return '\r'; + default: + return escaped; + } +} + +} // namespace xeger_internal diff --git a/web/backend/Database/Database.Seed/Database.Seed.csproj b/web/backend/Database/Database.Seed/Database.Seed.csproj index 9d11ce11..0d9b350c 100644 --- a/web/backend/Database/Database.Seed/Database.Seed.csproj +++ b/web/backend/Database/Database.Seed/Database.Seed.csproj @@ -10,8 +10,15 @@ - + + + + + diff --git a/web/backend/Database/Database.Seed/DatabaseHelpers.cs b/web/backend/Database/Database.Seed/DatabaseHelpers.cs new file mode 100644 index 00000000..574296b4 --- /dev/null +++ b/web/backend/Database/Database.Seed/DatabaseHelpers.cs @@ -0,0 +1,54 @@ +using Microsoft.Data.SqlClient; + +namespace Database.Seed; + +public class ConnectionStrings +{ + /// + /// Builds a SQL Server connection string from the DB_SERVER, DB_NAME, + /// DB_USER, DB_PASSWORD, and DB_TRUST_SERVER_CERTIFICATE + /// environment variables. + /// + /// A fully built SQL Server connection string. + /// + /// Thrown when DB_SERVER, DB_NAME, DB_USER, or DB_PASSWORD + /// is not set. + /// + private static string GetSqlServerConnectionString() + { + string server = + Environment.GetEnvironmentVariable("DB_SERVER") + ?? throw new InvalidOperationException("DB_SERVER environment variable is not set"); + + string dbName = + Environment.GetEnvironmentVariable("DB_NAME") + ?? throw new InvalidOperationException("DB_NAME environment variable is not set"); + + string user = + Environment.GetEnvironmentVariable("DB_USER") + ?? throw new InvalidOperationException("DB_USER environment variable is not set"); + + string password = + Environment.GetEnvironmentVariable("DB_PASSWORD") + ?? throw new InvalidOperationException("DB_PASSWORD environment variable is not set"); + + string trustServerCertificate = + Environment.GetEnvironmentVariable("DB_TRUST_SERVER_CERTIFICATE") ?? "True"; + + SqlConnectionStringBuilder builder = new() + { + DataSource = server, + InitialCatalog = dbName, + UserID = user, + Password = password, + TrustServerCertificate = bool.Parse(trustServerCertificate), + Encrypt = true, + }; + + return builder.ConnectionString; + } + + + public static string SqlServerConnectionString => GetSqlServerConnectionString(); + public static string SqliteConnectionString => "Data Source=seed.sqlite"; +} diff --git a/web/backend/Database/Database.Seed/ISeeder.cs b/web/backend/Database/Database.Seed/ISeeder.cs deleted file mode 100644 index 7c089389..00000000 --- a/web/backend/Database/Database.Seed/ISeeder.cs +++ /dev/null @@ -1,19 +0,0 @@ -using Microsoft.Data.SqlClient; - -namespace Database.Seed; - -/// -/// Defines a unit of seed data that can be applied to the database. Implementations -/// should be safe to run against an already-seeded database (e.g. by checking for -/// existing data before inserting) and may depend on data created by seeders that run -/// before them. -/// -internal interface ISeeder -{ - /// - /// Inserts this seeder's data into the database using the supplied open connection. - /// - /// An open connection to the target database. - /// A task that completes when seeding is finished. - Task SeedAsync(SqlConnection connection); -} diff --git a/web/backend/Database/Database.Seed/LocationSeeder.cs b/web/backend/Database/Database.Seed/LocationSeeder.cs deleted file mode 100644 index e3ba5c29..00000000 --- a/web/backend/Database/Database.Seed/LocationSeeder.cs +++ /dev/null @@ -1,341 +0,0 @@ -using System.Data; -using Microsoft.Data.SqlClient; - -namespace Database.Seed; - -/// -/// Seeds the location hierarchy (countries, states/provinces, and cities) used to -/// associate other entities (e.g. breweries, users) with a geographic location. -/// Countries must be seeded before states/provinces, which must be seeded before -/// cities, since each level references its parent by code. The underlying stored -/// procedures (USP_CreateCountry, USP_CreateStateProvince, -/// USP_CreateCity) are expected to be idempotent, so re-running this seeder -/// against an already-seeded database is safe. -/// -internal class LocationSeeder : ISeeder -{ - /// The set of countries to seed, identified by name and ISO 3166-1 code. - private static readonly IReadOnlyList<(string CountryName, string CountryCode)> Countries = - [ - ("Canada", "CA"), - ("Mexico", "MX"), - ("United States", "US"), - ]; - - /// - /// The set of states/provinces to seed, each identified by name, ISO 3166-2 code, - /// and the ISO 3166-1 code of its parent country. - /// - private static IReadOnlyList<( - string StateProvinceName, - string StateProvinceCode, - string CountryCode - )> States { get; } = - [ - ("Alabama", "US-AL", "US"), - ("Alaska", "US-AK", "US"), - ("Arizona", "US-AZ", "US"), - ("Arkansas", "US-AR", "US"), - ("California", "US-CA", "US"), - ("Colorado", "US-CO", "US"), - ("Connecticut", "US-CT", "US"), - ("Delaware", "US-DE", "US"), - ("Florida", "US-FL", "US"), - ("Georgia", "US-GA", "US"), - ("Hawaii", "US-HI", "US"), - ("Idaho", "US-ID", "US"), - ("Illinois", "US-IL", "US"), - ("Indiana", "US-IN", "US"), - ("Iowa", "US-IA", "US"), - ("Kansas", "US-KS", "US"), - ("Kentucky", "US-KY", "US"), - ("Louisiana", "US-LA", "US"), - ("Maine", "US-ME", "US"), - ("Maryland", "US-MD", "US"), - ("Massachusetts", "US-MA", "US"), - ("Michigan", "US-MI", "US"), - ("Minnesota", "US-MN", "US"), - ("Mississippi", "US-MS", "US"), - ("Missouri", "US-MO", "US"), - ("Montana", "US-MT", "US"), - ("Nebraska", "US-NE", "US"), - ("Nevada", "US-NV", "US"), - ("New Hampshire", "US-NH", "US"), - ("New Jersey", "US-NJ", "US"), - ("New Mexico", "US-NM", "US"), - ("New York", "US-NY", "US"), - ("North Carolina", "US-NC", "US"), - ("North Dakota", "US-ND", "US"), - ("Ohio", "US-OH", "US"), - ("Oklahoma", "US-OK", "US"), - ("Oregon", "US-OR", "US"), - ("Pennsylvania", "US-PA", "US"), - ("Rhode Island", "US-RI", "US"), - ("South Carolina", "US-SC", "US"), - ("South Dakota", "US-SD", "US"), - ("Tennessee", "US-TN", "US"), - ("Texas", "US-TX", "US"), - ("Utah", "US-UT", "US"), - ("Vermont", "US-VT", "US"), - ("Virginia", "US-VA", "US"), - ("Washington", "US-WA", "US"), - ("West Virginia", "US-WV", "US"), - ("Wisconsin", "US-WI", "US"), - ("Wyoming", "US-WY", "US"), - ("District of Columbia", "US-DC", "US"), - ("Puerto Rico", "US-PR", "US"), - ("U.S. Virgin Islands", "US-VI", "US"), - ("Guam", "US-GU", "US"), - ("Northern Mariana Islands", "US-MP", "US"), - ("American Samoa", "US-AS", "US"), - ("Ontario", "CA-ON", "CA"), - ("Québec", "CA-QC", "CA"), - ("Nova Scotia", "CA-NS", "CA"), - ("New Brunswick", "CA-NB", "CA"), - ("Manitoba", "CA-MB", "CA"), - ("British Columbia", "CA-BC", "CA"), - ("Prince Edward Island", "CA-PE", "CA"), - ("Saskatchewan", "CA-SK", "CA"), - ("Alberta", "CA-AB", "CA"), - ("Newfoundland and Labrador", "CA-NL", "CA"), - ("Northwest Territories", "CA-NT", "CA"), - ("Yukon", "CA-YT", "CA"), - ("Nunavut", "CA-NU", "CA"), - ("Aguascalientes", "MX-AGU", "MX"), - ("Baja California", "MX-BCN", "MX"), - ("Baja California Sur", "MX-BCS", "MX"), - ("Campeche", "MX-CAM", "MX"), - ("Chiapas", "MX-CHP", "MX"), - ("Chihuahua", "MX-CHH", "MX"), - ("Coahuila de Zaragoza", "MX-COA", "MX"), - ("Colima", "MX-COL", "MX"), - ("Durango", "MX-DUR", "MX"), - ("Guanajuato", "MX-GUA", "MX"), - ("Guerrero", "MX-GRO", "MX"), - ("Hidalgo", "MX-HID", "MX"), - ("Jalisco", "MX-JAL", "MX"), - ("México State", "MX-MEX", "MX"), - ("Michoacán de Ocampo", "MX-MIC", "MX"), - ("Morelos", "MX-MOR", "MX"), - ("Nayarit", "MX-NAY", "MX"), - ("Nuevo León", "MX-NLE", "MX"), - ("Oaxaca", "MX-OAX", "MX"), - ("Puebla", "MX-PUE", "MX"), - ("Querétaro", "MX-QUE", "MX"), - ("Quintana Roo", "MX-ROO", "MX"), - ("San Luis Potosí", "MX-SLP", "MX"), - ("Sinaloa", "MX-SIN", "MX"), - ("Sonora", "MX-SON", "MX"), - ("Tabasco", "MX-TAB", "MX"), - ("Tamaulipas", "MX-TAM", "MX"), - ("Tlaxcala", "MX-TLA", "MX"), - ("Veracruz de Ignacio de la Llave", "MX-VER", "MX"), - ("Yucatán", "MX-YUC", "MX"), - ("Zacatecas", "MX-ZAC", "MX"), - ("Ciudad de México", "MX-CMX", "MX"), - ]; - - /// - /// The set of cities to seed, each identified by name and the ISO 3166-2 code of its - /// parent state/province. - /// - private static IReadOnlyList<(string StateProvinceCode, string CityName)> Cities { get; } = - [ - ("US-CA", "Los Angeles"), - ("US-CA", "San Diego"), - ("US-CA", "San Francisco"), - ("US-CA", "Sacramento"), - ("US-TX", "Houston"), - ("US-TX", "Dallas"), - ("US-TX", "Austin"), - ("US-TX", "San Antonio"), - ("US-FL", "Miami"), - ("US-FL", "Orlando"), - ("US-FL", "Tampa"), - ("US-NY", "New York"), - ("US-NY", "Buffalo"), - ("US-NY", "Rochester"), - ("US-IL", "Chicago"), - ("US-IL", "Springfield"), - ("US-PA", "Philadelphia"), - ("US-PA", "Pittsburgh"), - ("US-AZ", "Phoenix"), - ("US-AZ", "Tucson"), - ("US-CO", "Denver"), - ("US-CO", "Colorado Springs"), - ("US-MA", "Boston"), - ("US-MA", "Worcester"), - ("US-WA", "Seattle"), - ("US-WA", "Spokane"), - ("US-GA", "Atlanta"), - ("US-GA", "Savannah"), - ("US-NV", "Las Vegas"), - ("US-NV", "Reno"), - ("US-MI", "Detroit"), - ("US-MI", "Grand Rapids"), - ("US-MN", "Minneapolis"), - ("US-MN", "Saint Paul"), - ("US-OH", "Columbus"), - ("US-OH", "Cleveland"), - ("US-OR", "Portland"), - ("US-OR", "Salem"), - ("US-TN", "Nashville"), - ("US-TN", "Memphis"), - ("US-VA", "Richmond"), - ("US-VA", "Virginia Beach"), - ("US-MD", "Baltimore"), - ("US-MD", "Frederick"), - ("US-DC", "Washington"), - ("US-UT", "Salt Lake City"), - ("US-UT", "Provo"), - ("US-LA", "New Orleans"), - ("US-LA", "Baton Rouge"), - ("US-KY", "Louisville"), - ("US-KY", "Lexington"), - ("US-IA", "Des Moines"), - ("US-IA", "Cedar Rapids"), - ("US-OK", "Oklahoma City"), - ("US-OK", "Tulsa"), - ("US-NE", "Omaha"), - ("US-NE", "Lincoln"), - ("US-MO", "Kansas City"), - ("US-MO", "St. Louis"), - ("US-NC", "Charlotte"), - ("US-NC", "Raleigh"), - ("US-SC", "Columbia"), - ("US-SC", "Charleston"), - ("US-WI", "Milwaukee"), - ("US-WI", "Madison"), - ("US-MN", "Duluth"), - ("US-AK", "Anchorage"), - ("US-HI", "Honolulu"), - ("CA-ON", "Toronto"), - ("CA-ON", "Ottawa"), - ("CA-QC", "Montréal"), - ("CA-QC", "Québec City"), - ("CA-BC", "Vancouver"), - ("CA-BC", "Victoria"), - ("CA-AB", "Calgary"), - ("CA-AB", "Edmonton"), - ("CA-MB", "Winnipeg"), - ("CA-NS", "Halifax"), - ("CA-SK", "Saskatoon"), - ("CA-SK", "Regina"), - ("CA-NB", "Moncton"), - ("CA-NB", "Saint John"), - ("CA-PE", "Charlottetown"), - ("CA-NL", "St. John's"), - ("CA-ON", "Hamilton"), - ("CA-ON", "London"), - ("CA-QC", "Gatineau"), - ("CA-QC", "Laval"), - ("CA-BC", "Kelowna"), - ("CA-AB", "Red Deer"), - ("CA-MB", "Brandon"), - ("MX-CMX", "Ciudad de México"), - ("MX-JAL", "Guadalajara"), - ("MX-NLE", "Monterrey"), - ("MX-PUE", "Puebla"), - ("MX-ROO", "Cancún"), - ("MX-GUA", "Guanajuato"), - ("MX-MIC", "Morelia"), - ("MX-BCN", "Tijuana"), - ("MX-JAL", "Zapopan"), - ("MX-NLE", "San Nicolás"), - ("MX-CAM", "Campeche"), - ("MX-TAB", "Villahermosa"), - ("MX-VER", "Veracruz"), - ("MX-OAX", "Oaxaca"), - ("MX-SLP", "San Luis Potosí"), - ("MX-CHH", "Chihuahua"), - ("MX-AGU", "Aguascalientes"), - ("MX-MEX", "Toluca"), - ("MX-COA", "Saltillo"), - ("MX-BCS", "La Paz"), - ("MX-NAY", "Tepic"), - ("MX-ZAC", "Zacatecas"), - ]; - - /// - /// Seeds all countries, then states/provinces, then cities, in that order, so that - /// each level's parent reference already exists by the time it is created. - /// - /// An open connection to the target database. - /// A task that completes when all locations have been seeded. - public async Task SeedAsync(SqlConnection connection) - { - foreach ((string countryName, string countryCode) in Countries) - await CreateCountryAsync(connection, countryName, countryCode); - - foreach ((string stateProvinceName, string stateProvinceCode, string countryCode) in States) - await CreateStateProvinceAsync( - connection, - stateProvinceName, - stateProvinceCode, - countryCode - ); - - foreach ((string stateProvinceCode, string cityName) in Cities) - await CreateCityAsync(connection, cityName, stateProvinceCode); - } - - /// Creates a single country by invoking the dbo.USP_CreateCountry stored procedure. - /// An open connection to the target database. - /// The display name of the country. - /// The ISO 3166-1 code of the country. - /// A task that completes when the country has been created. - private static async Task CreateCountryAsync( - SqlConnection connection, - string countryName, - string countryCode - ) - { - await using SqlCommand command = new("dbo.USP_CreateCountry", connection); - command.CommandType = CommandType.StoredProcedure; - command.Parameters.AddWithValue("@CountryName", countryName); - command.Parameters.AddWithValue("@ISO3166_1", countryCode); - - await command.ExecuteNonQueryAsync(); - } - - /// Creates a single state/province by invoking the dbo.USP_CreateStateProvince stored procedure. - /// An open connection to the target database. - /// The display name of the state/province. - /// The ISO 3166-2 code of the state/province. - /// The ISO 3166-1 code of the parent country, which must already exist. - /// A task that completes when the state/province has been created. - private static async Task CreateStateProvinceAsync( - SqlConnection connection, - string stateProvinceName, - string stateProvinceCode, - string countryCode - ) - { - await using SqlCommand command = new("dbo.USP_CreateStateProvince", connection); - command.CommandType = CommandType.StoredProcedure; - command.Parameters.AddWithValue("@StateProvinceName", stateProvinceName); - command.Parameters.AddWithValue("@ISO3166_2", stateProvinceCode); - command.Parameters.AddWithValue("@CountryCode", countryCode); - - await command.ExecuteNonQueryAsync(); - } - - /// Creates a single city by invoking the dbo.USP_CreateCity stored procedure. - /// An open connection to the target database. - /// The display name of the city. - /// The ISO 3166-2 code of the parent state/province, which must already exist. - /// A task that completes when the city has been created. - private static async Task CreateCityAsync( - SqlConnection connection, - string cityName, - string stateProvinceCode - ) - { - await using SqlCommand command = new("dbo.USP_CreateCity", connection); - command.CommandType = CommandType.StoredProcedure; - command.Parameters.AddWithValue("@CityName", cityName); - command.Parameters.AddWithValue("@StateProvinceCode", stateProvinceCode); - - await command.ExecuteNonQueryAsync(); - } -} diff --git a/web/backend/Database/Database.Seed/Program.cs b/web/backend/Database/Database.Seed/Program.cs deleted file mode 100644 index fa048de0..00000000 --- a/web/backend/Database/Database.Seed/Program.cs +++ /dev/null @@ -1,106 +0,0 @@ -using Database.Seed; -using Microsoft.Data.SqlClient; - -// Entry point for the database seeding utility. Connects to the target database -// (retrying on transient failures), then runs each registered ISeeder in order to -// populate location and user data. - -/// -/// Builds a SQL Server connection string from the DB_SERVER, DB_NAME, -/// DB_USER, DB_PASSWORD, and DB_TRUST_SERVER_CERTIFICATE -/// environment variables. -/// -/// A fully built SQL Server connection string. -/// -/// Thrown when DB_SERVER, DB_NAME, DB_USER, or DB_PASSWORD -/// is not set. -/// -string BuildConnectionString() -{ - string server = - Environment.GetEnvironmentVariable("DB_SERVER") - ?? throw new InvalidOperationException("DB_SERVER environment variable is not set"); - - string dbName = - Environment.GetEnvironmentVariable("DB_NAME") - ?? throw new InvalidOperationException("DB_NAME environment variable is not set"); - - string user = - Environment.GetEnvironmentVariable("DB_USER") - ?? throw new InvalidOperationException("DB_USER environment variable is not set"); - - string password = - Environment.GetEnvironmentVariable("DB_PASSWORD") - ?? throw new InvalidOperationException("DB_PASSWORD environment variable is not set"); - - string trustServerCertificate = - Environment.GetEnvironmentVariable("DB_TRUST_SERVER_CERTIFICATE") ?? "True"; - - SqlConnectionStringBuilder builder = new() - { - DataSource = server, - InitialCatalog = dbName, - UserID = user, - Password = password, - TrustServerCertificate = bool.Parse(trustServerCertificate), - Encrypt = true, - }; - - return builder.ConnectionString; -} - -try -{ - string connectionString = BuildConnectionString(); - - Console.WriteLine("Attempting to connect to database..."); - - // Retry logic for database connection - SqlConnection? connection = null; - int maxRetries = 10; - int retryDelayMs = 2000; - - for (int attempt = 1; attempt <= maxRetries; attempt++) - try - { - connection = new SqlConnection(connectionString); - await connection.OpenAsync(); - Console.WriteLine($"Connected to database successfully on attempt {attempt}."); - break; - } - catch (SqlException ex) when (attempt < maxRetries) - { - Console.WriteLine($"Connection attempt {attempt}/{maxRetries} failed: {ex.Message}"); - Console.WriteLine($"Retrying in {retryDelayMs}ms..."); - await Task.Delay(retryDelayMs); - connection?.Dispose(); - connection = null; - } - - if (connection == null) - throw new Exception($"Failed to connect to database after {maxRetries} attempts."); - - Console.WriteLine("Starting seeding..."); - - using (connection) - { - ISeeder[] seeders = [new LocationSeeder(), new UserSeeder()]; - - foreach (ISeeder seeder in seeders) - { - Console.WriteLine($"Seeding {seeder.GetType().Name}..."); - await seeder.SeedAsync(connection); - Console.WriteLine($"{seeder.GetType().Name} seeded."); - } - - Console.WriteLine("Seed completed successfully."); - } - - return 0; -} -catch (Exception ex) -{ - Console.Error.WriteLine("Seed failed:"); - Console.Error.WriteLine(ex); - return 1; -} diff --git a/web/backend/Database/Database.Seed/UserSeeder.cs b/web/backend/Database/Database.Seed/UserSeeder.cs deleted file mode 100644 index 5a3a312d..00000000 --- a/web/backend/Database/Database.Seed/UserSeeder.cs +++ /dev/null @@ -1,303 +0,0 @@ -using System.Data; -using System.Security.Cryptography; -using System.Text; -using idunno.Password; -using Konscious.Security.Cryptography; -using Microsoft.Data.SqlClient; - -namespace Database.Seed; - -/// -/// Seeds user accounts, credentials, and verification records. Creates one fixed -/// "Test User" account (test.user@thebiergarten.app / password "password") -/// for testing, followed by a randomly-generated account for each name in -/// , each with a random password and date of birth. Skips -/// adding a verification record for a user that already has one, so this seeder is -/// safe to re-run, though re-running will still attempt to register (and may fail on) -/// users that already exist. -/// -internal class UserSeeder : ISeeder -{ - /// The first/last name pairs used to generate seed user accounts. - private static readonly IReadOnlyList<(string FirstName, string LastName)> SeedNames = - [ - ("Aarya", "Mathews"), - ("Aiden", "Wells"), - ("Aleena", "Gonzalez"), - ("Alessandra", "Nelson"), - ("Amari", "Tucker"), - ("Ameer", "Huff"), - ("Amirah", "Hicks"), - ("Analia", "Dominguez"), - ("Anne", "Jenkins"), - ("Apollo", "Davis"), - ("Arianna", "White"), - ("Aubree", "Moore"), - ("Aubrielle", "Raymond"), - ("Aydin", "Odom"), - ("Bowen", "Casey"), - ("Brock", "Huber"), - ("Caiden", "Strong"), - ("Cecilia", "Rosales"), - ("Celeste", "Barber"), - ("Chance", "Small"), - ("Clara", "Roberts"), - ("Collins", "Brandt"), - ("Damir", "Wallace"), - ("Declan", "Crawford"), - ("Dennis", "Decker"), - ("Dylan", "Lang"), - ("Eliza", "Kane"), - ("Elle", "Poole"), - ("Elliott", "Miles"), - ("Emelia", "Lucas"), - ("Emilia", "Simpson"), - ("Emmett", "Lugo"), - ("Ethan", "Stephens"), - ("Etta", "Woods"), - ("Gael", "Moran"), - ("Grant", "Benson"), - ("Gwen", "James"), - ("Huxley", "Chen"), - ("Isabella", "Fisher"), - ("Ivan", "Mathis"), - ("Jamir", "McMillan"), - ("Jaxson", "Shields"), - ("Jimmy", "Richmond"), - ("Josiah", "Flores"), - ("Kaden", "Enriquez"), - ("Kai", "Lawson"), - ("Karsyn", "Adkins"), - ("Karsyn", "Proctor"), - ("Kayden", "Henson"), - ("Kaylie", "Spears"), - ("Kinslee", "Jones"), - ("Kora", "Guerra"), - ("Lane", "Skinner"), - ("Laylani", "Christian"), - ("Ledger", "Carroll"), - ("Leilany", "Small"), - ("Leland", "McCall"), - ("Leonard", "Calhoun"), - ("Levi", "Ochoa"), - ("Lillie", "Vang"), - ("Lola", "Sheppard"), - ("Luciana", "Poole"), - ("Maddox", "Hughes"), - ("Mara", "Blackwell"), - ("Marcellus", "Bartlett"), - ("Margo", "Koch"), - ("Maurice", "Gibson"), - ("Maxton", "Dodson"), - ("Mia", "Parrish"), - ("Millie", "Fuentes"), - ("Nellie", "Villanueva"), - ("Nicolas", "Mata"), - ("Nicolas", "Miller"), - ("Oakleigh", "Foster"), - ("Octavia", "Pierce"), - ("Paisley", "Allison"), - ("Quincy", "Andersen"), - ("Quincy", "Frazier"), - ("Raiden", "Roberts"), - ("Raquel", "Lara"), - ("Rudy", "McIntosh"), - ("Salvador", "Stein"), - ("Samantha", "Dickson"), - ("Solomon", "Richards"), - ("Sylvia", "Hanna"), - ("Talia", "Trujillo"), - ("Thalia", "Farrell"), - ("Trent", "Mayo"), - ("Trinity", "Cummings"), - ("Ty", "Perry"), - ("Tyler", "Romero"), - ("Valeria", "Pierce"), - ("Vance", "Neal"), - ("Whitney", "Bell"), - ("Wilder", "Graves"), - ("William", "Logan"), - ("Zara", "Wilkinson"), - ("Zaria", "Gibson"), - ("Zion", "Watkins"), - ("Zoie", "Armstrong"), - ]; - - /// - /// Registers a fixed test user account followed by one randomly-generated account - /// per entry in , adding a user verification record for each - /// newly created account that does not already have one. Progress counts are - /// written to the console on completion. - /// - /// An open connection to the target database. - /// A task that completes when all users have been seeded. - public async Task SeedAsync(SqlConnection connection) - { - PasswordGenerator generator = new(); - Random rng = new(); - int createdUsers = 0; - int createdCredentials = 0; - int createdVerifications = 0; - - // create a known user for testing purposes - { - const string firstName = "Test"; - const string lastName = "User"; - const string email = "test.user@thebiergarten.app"; - DateTime dob = new(1985, 03, 01); - string hash = GeneratePasswordHash("password"); - - await RegisterUserAsync( - connection, - $"{firstName}.{lastName}", - firstName, - lastName, - dob, - email, - hash - ); - } - foreach ((string firstName, string lastName) in SeedNames) - { - // prepare user fields - string username = $"{firstName[0]}.{lastName}"; - string email = $"{firstName}.{lastName}@thebiergarten.app"; - DateTime dob = GenerateDateOfBirth(rng); - - // generate a password and hash it - string pwd = generator.Generate(64, 10, 10); - string hash = GeneratePasswordHash(pwd); - - // register the user (creates account + credential) - Guid id = await RegisterUserAsync( - connection, - username, - firstName, - lastName, - dob, - email, - hash - ); - createdUsers++; - createdCredentials++; - - // add user verification - if (await HasUserVerificationAsync(connection, id)) - continue; - - await AddUserVerificationAsync(connection, id); - createdVerifications++; - } - - Console.WriteLine($"Created {createdUsers} user accounts."); - Console.WriteLine($"Added {createdCredentials} user credentials."); - Console.WriteLine($"Added {createdVerifications} user verifications."); - } - - /// - /// Registers a new user account and its credential by invoking the - /// dbo.USP_RegisterUser stored procedure. - /// - /// An open connection to the target database. - /// The unique username for the account. - /// The user's first name. - /// The user's last name. - /// The user's date of birth. - /// The user's email address. - /// The salted password hash, as produced by . - /// The newly created user account's identifier. - private static async Task RegisterUserAsync( - SqlConnection connection, - string username, - string firstName, - string lastName, - DateTime dateOfBirth, - string email, - string hash - ) - { - await using SqlCommand command = new("dbo.USP_RegisterUser", connection); - command.CommandType = CommandType.StoredProcedure; - - command.Parameters.Add("@Username", SqlDbType.VarChar, 64).Value = username; - command.Parameters.Add("@FirstName", SqlDbType.NVarChar, 128).Value = firstName; - command.Parameters.Add("@LastName", SqlDbType.NVarChar, 128).Value = lastName; - command.Parameters.Add("@DateOfBirth", SqlDbType.DateTime).Value = dateOfBirth; - command.Parameters.Add("@Email", SqlDbType.VarChar, 128).Value = email; - command.Parameters.Add("@Hash", SqlDbType.NVarChar, -1).Value = hash; - - object? result = await command.ExecuteScalarAsync(); - - return (Guid)result!; - } - - /// - /// Hashes a plaintext password using Argon2id with a randomly generated 16-byte - /// salt, a degree of parallelism matching the available processor count, 64 MB of - /// memory, and 4 iterations. - /// - /// The plaintext password to hash. - /// A string containing the base64-encoded salt and hash, separated by a colon. - private static string GeneratePasswordHash(string pwd) - { - byte[] salt = RandomNumberGenerator.GetBytes(16); - - Argon2id argon2 = new(Encoding.UTF8.GetBytes(pwd)) - { - Salt = salt, - DegreeOfParallelism = Math.Max(Environment.ProcessorCount, 1), - MemorySize = 65536, - Iterations = 4, - }; - - byte[] hash = argon2.GetBytes(32); - return $"{Convert.ToBase64String(salt)}:{Convert.ToBase64String(hash)}"; - } - - /// Checks whether a user verification record already exists for the given user account. - /// An open connection to the target database. - /// The user account identifier to check. - /// true if a verification record already exists; otherwise false. - private static async Task HasUserVerificationAsync( - SqlConnection connection, - Guid userAccountId - ) - { - const string sql = """ - SELECT 1 - FROM dbo.UserVerification - WHERE UserAccountId = @UserAccountId; - """; - await using SqlCommand command = new(sql, connection); - command.Parameters.AddWithValue("@UserAccountId", userAccountId); - object? result = await command.ExecuteScalarAsync(); - return result is not null; - } - - /// Creates a user verification record by invoking the dbo.USP_CreateUserVerification stored procedure. - /// An open connection to the target database. - /// The identifier of the user account to verify. - /// A task that completes when the verification record has been created. - private static async Task AddUserVerificationAsync(SqlConnection connection, Guid userAccountId) - { - await using SqlCommand command = new("dbo.USP_CreateUserVerification", connection); - command.CommandType = CommandType.StoredProcedure; - command.Parameters.AddWithValue("@UserAccountID_", userAccountId); - - await command.ExecuteNonQueryAsync(); - } - - /// - /// Generates a random date of birth corresponding to an age between 19 and 48 - /// years (inclusive), with a random day offset within that birth year. - /// - /// The random number source to use. - /// A randomly generated date of birth. - private static DateTime GenerateDateOfBirth(Random random) - { - int age = 19 + random.Next(0, 30); - DateTime baseDate = DateTime.UtcNow.Date.AddYears(-age); - int offsetDays = random.Next(0, 365); - return baseDate.AddDays(-offsetDays); - } -} diff --git a/web/backend/Database/Program.cs b/web/backend/Database/Program.cs new file mode 100644 index 00000000..f486b067 --- /dev/null +++ b/web/backend/Database/Program.cs @@ -0,0 +1,18 @@ +namespace Database.Seed; + + +public interface IExportDatabase +{ + void SeedLocations(IEnumerable locations); + void SeedUsers(IEnumerable users); + + void SeedBreweries(IEnumerable breweries); +} + +public interface IImportDatabase +{ + IEnumerable GetLocations(); + IEnumerable GetUsers(); + + IEnumerable GetBreweries(); +} diff --git a/web/backend/Infrastructure/Infrastructure.Jwt/JwtInfrastructure.cs b/web/backend/Infrastructure/Infrastructure.Jwt/JwtInfrastructure.cs index 72837bb9..f8707b34 100644 --- a/web/backend/Infrastructure/Infrastructure.Jwt/JwtInfrastructure.cs +++ b/web/backend/Infrastructure/Infrastructure.Jwt/JwtInfrastructure.cs @@ -88,7 +88,7 @@ public class JwtInfrastructure : ITokenInfrastructure return new ClaimsPrincipal(result.ClaimsIdentity); } - catch (Exception e) + catch (Exception) { throw new UnauthorizedException("Invalid token"); } diff --git a/web/backend/Infrastructure/Infrastructure.Sql/Infrastructure.Sql.csproj b/web/backend/Infrastructure/Infrastructure.Sql/Infrastructure.Sql.csproj index 7ff048e9..dcc69bba 100644 --- a/web/backend/Infrastructure/Infrastructure.Sql/Infrastructure.Sql.csproj +++ b/web/backend/Infrastructure/Infrastructure.Sql/Infrastructure.Sql.csproj @@ -6,7 +6,7 @@ Infrastructure.Sql - +