From 22b3f2b3f9ed5a21862a24c8d743c3b2142b2d64 Mon Sep 17 00:00:00 2001 From: Aaron Po Date: Tue, 23 Jun 2026 00:38:53 -0400 Subject: [PATCH] cleanup gbnf grammars --- .../includes/data_generation/json_grammars.h | 44 +++++++++++++++++++ .../generate_users.cc | 5 ++- .../data_generation/llama/generate_brewery.cc | 14 +----- .../data_generation/llama/generate_user.cc | 13 +----- 4 files changed, 49 insertions(+), 27 deletions(-) create mode 100644 tooling/pipeline/includes/data_generation/json_grammars.h diff --git a/tooling/pipeline/includes/data_generation/json_grammars.h b/tooling/pipeline/includes/data_generation/json_grammars.h new file mode 100644 index 0000000..40bd338 --- /dev/null +++ b/tooling/pipeline/includes/data_generation/json_grammars.h @@ -0,0 +1,44 @@ +#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_JSON_GRAMMARS_H_ +#define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_JSON_GRAMMARS_H_ + +/** + * @file data_generation/json_grammars.h + * @brief GBNF grammars constraining structured JSON output from + * LlamaGenerator inference calls. + */ + +#include + +// GBNF grammar for structured user JSON output. +inline constexpr std::string_view kUserJsonGrammar = R"json_user( +root ::= thought-block "{" ws + "\"username\"" ws ":" ws string ws "," ws + "\"bio\"" ws ":" ws string ws "," ws + "\"activity_weight\"" ws ":" ws number ws + "}" ws +thought-block ::= [^{]* +ws ::= [ \t\n\r]* +string ::= "\"" char+ "\"" +char ::= [^"\\\x7F\x00-\x1F] | [\\] escape +escape ::= ["\\/bfnrt] | "u" hex hex hex hex +hex ::= [0-9a-fA-F] +number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? +)json_user"; + +// GBNF grammar for structured brewery JSON output. +inline constexpr std::string_view kBreweryJsonGrammar = R"json_brewery( +root ::= thought-block "{" ws + "\"name_en\"" ws ":" ws string ws "," ws + "\"description_en\"" ws ":" ws string ws "," ws + "\"name_local\"" ws ":" ws string ws "," ws + "\"description_local\"" ws ":" ws string ws + "}" ws +thought-block ::= [^{]* +ws ::= [ \t\n\r]* +string ::= "\"" char+ "\"" +char ::= [^"\\\x7F\x00-\x1F] | [\\] escape +escape ::= ["\\/bfnrt] | "u" hex hex hex hex +hex ::= [0-9a-fA-F] +)json_brewery"; + +#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_JSON_GRAMMARS_H_ diff --git a/tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc b/tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc index 1b94e1e..2a8fb23 100644 --- a/tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc +++ b/tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc @@ -110,15 +110,16 @@ void BiergartenPipelineOrchestrator::GenerateUsers( std::uniform_int_distribution persona_dist(0, personas.size() - 1); generated_users_.clear(); - std::unordered_set used_email_local_parts; size_t skipped_count = 0; size_t export_failed_count = 0; const auto generate_record = - [this, &rng, &used_email_local_parts, &skipped_count]( + [this, &rng, &skipped_count]( const EnrichedCity& city, const UserPersona& persona, const Name& sampled_name) -> std::optional { try { + std::unordered_set used_email_local_parts; + const UserResult user = generator_->GenerateUser(city, persona, sampled_name); diff --git a/tooling/pipeline/src/data_generation/llama/generate_brewery.cc b/tooling/pipeline/src/data_generation/llama/generate_brewery.cc index 0589a43..30c4ac6 100644 --- a/tooling/pipeline/src/data_generation/llama/generate_brewery.cc +++ b/tooling/pipeline/src/data_generation/llama/generate_brewery.cc @@ -12,6 +12,7 @@ #include #include +#include "data_generation/json_grammars.h" #include "data_generation/llama_generator.h" #include "data_generation/llama_generator_helpers.h" @@ -32,19 +33,6 @@ static std::string FormatLocalLanguageCodes( return formatted; } -// GBNF grammar for structured brewery JSON output. -// @TODO move to a separate gbnf file if it grows in complexity or is shared -// across modules. -static constexpr std::string_view kBreweryJsonGrammar = R"json_brewery( -root ::= thought-block "{" ws "\"name_en\"" ws ":" ws string ws "," ws "\"description_en\"" ws ":" ws string ws "," ws "\"name_local\"" ws ":" ws string ws "," ws "\"description_local\"" ws ":" ws string ws "}" ws -thought-block ::= [^{]* -ws ::= [ \t\n\r]* -string ::= "\"" char+ "\"" -char ::= [^"\\\x7F\x00-\x1F] | [\\] escape -escape ::= ["\\/bfnrt] | "u" hex hex hex hex -hex ::= [0-9a-fA-F] -)json_brewery"; - static constexpr int kBreweryInitialMaxTokens = 2800; BreweryResult LlamaGenerator::GenerateBrewery( diff --git a/tooling/pipeline/src/data_generation/llama/generate_user.cc b/tooling/pipeline/src/data_generation/llama/generate_user.cc index 8cf7b53..63adb82 100644 --- a/tooling/pipeline/src/data_generation/llama/generate_user.cc +++ b/tooling/pipeline/src/data_generation/llama/generate_user.cc @@ -10,21 +10,10 @@ #include #include +#include "data_generation/json_grammars.h" #include "data_generation/llama_generator.h" #include "data_generation/llama_generator_helpers.h" -// GBNF grammar for structured user JSON output. -static constexpr std::string_view kUserJsonGrammar = R"json_user( -root ::= thought-block "{" ws "\"username\"" ws ":" ws string ws "," ws "\"bio\"" ws ":" ws string ws "," ws "\"activity_weight\"" ws ":" ws number ws "}" ws -thought-block ::= [^{]* -ws ::= [ \t\n\r]* -string ::= "\"" char+ "\"" -char ::= [^"\\\x7F\x00-\x1F] | [\\] escape -escape ::= ["\\/bfnrt] | "u" hex hex hex hex -hex ::= [0-9a-fA-F] -number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? -)json_user"; - static constexpr int kUserInitialMaxTokens = 1200; UserResult LlamaGenerator::GenerateUser(const EnrichedCity& city,