Pipeline: Add LLM and mocked user generation to the pipeline (#230)

This commit is contained in:
2026-07-01 07:47:24 -04:00
committed by GitHub
parent 880f73e004
commit 2b8a900d12
90 changed files with 57254 additions and 800 deletions

View File

@@ -93,6 +93,77 @@ while (For each sampled Location?) is (Remaining cities)
endif
endwhile (Done)
|#EAF0E8|BiergartenPipelineOrchestrator|
:GenerateUsers(enriched_cities);
|#E2EBDC|JsonLoader|
:LoadPersonas("personas.json");
:LoadForenamesByCountry("forenames-by-country.json");
:LoadSurnamesByCountry("surnames-by-country.json");
note right
Each call is cached after its first parse.
MockCuratedDataService (--mocked) returns a
fixed in-memory dataset instead and skips
these three JSON files entirely.
end note
|#EAF0E8|BiergartenPipelineOrchestrator|
while (For each EnrichedCity?) is (Remaining cities)
:SampleName(forenames_by_country,\n surnames_by_country, location.iso3166_1);
if (Names available for country?) then (no)
:Log warning, skip city;
else (yes)
:Sample UserPersona (uniform random);
|#E5EDE1|DataGenerator|
if (Generator Mode) then (MockGenerator)
:DeterministicHash & Format;
else (LlamaGenerator)
:prompt_directory_->Load("USER_GENERATION");
note right
Resolves to USER_GENERATION.md inside --prompt-dir.
end note
repeat
:Infer(system_prompt, user_prompt, max_tokens, kUserJsonGrammar);
:ValidateUserJson(raw, user);
if (Is JSON Valid?) then (yes)
break
else (no)
:Attempt++;
:Append validation error to retry prompt;
endif
repeat while (Attempt < 3?) is (yes)
endif
|#EAF0E8|BiergartenPipelineOrchestrator|
if (Generation successful?) then (yes)
:BuildEmail(name, used_email_local_parts);
:GenerateDateOfBirth(rng);
note right
email and date_of_birth are generated
programmatically, never LLM-authored.
end note
|#E0EAE0|SqliteExportService|
:ProcessRecord(UserRecord);
if (Location in cache?) then (yes)
:Reuse location_id;
else (no)
:Insert Location & Cache ID;
endif
:Insert User (FK: location_id);
if (Exception caught during insert?) then (yes)
|#EAF0E8|BiergartenPipelineOrchestrator|
:Log warning "SQLite export failed";
else (no)
endif
else (no)
:Log warning "Generation failed, skipping...";
endif
endif
endwhile (Done)
|#EAF0E8|BiergartenPipelineOrchestrator|
:GenerateBreweries(enriched_cities);