AaronPo/the-biergarten-app
1
0
Fork 0
mirror of https://github.com/aaronpo97/the-biergarten-app.git synced 2026-07-16 17:47:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

cleanup gbnf grammars

Browse Source
This commit is contained in:
Aaron Po
2026-06-23 00:38:53 -04:00
parent ac701d661a
commit 22b3f2b3f9
4 changed files with 49 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
tooling/pipeline/includes/data_generation/json_grammars.h Normal file
View File

@@ -0,0 +1,44 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_JSON_GRAMMARS_H_
#define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_JSON_GRAMMARS_H_
/**
* @file data_generation/json_grammars.h
* @brief GBNF grammars constraining structured JSON output from
* LlamaGenerator inference calls.
*/
#include <string_view>
// GBNF grammar for structured user JSON output.
inline constexpr std::string_view kUserJsonGrammar = R"json_user(
root ::= thought-block "{" ws
"\"username\"" ws ":" ws string ws "," ws
"\"bio\"" ws ":" ws string ws "," ws
"\"activity_weight\"" ws ":" ws number ws
"}" ws
thought-block ::= [^{]*
ws ::= [ \t\n\r]*
string ::= "\"" char+ "\""
char ::= [^"\\\x7F\x00-\x1F] | [\\] escape
escape ::= ["\\/bfnrt] | "u" hex hex hex hex
hex ::= [0-9a-fA-F]
number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)?
)json_user";
// GBNF grammar for structured brewery JSON output.
inline constexpr std::string_view kBreweryJsonGrammar = R"json_brewery(
root ::= thought-block "{" ws
"\"name_en\"" ws ":" ws string ws "," ws
"\"description_en\"" ws ":" ws string ws "," ws
"\"name_local\"" ws ":" ws string ws "," ws
"\"description_local\"" ws ":" ws string ws
"}" ws
thought-block ::= [^{]*
ws ::= [ \t\n\r]*
string ::= "\"" char+ "\""
char ::= [^"\\\x7F\x00-\x1F] | [\\] escape
escape ::= ["\\/bfnrt] | "u" hex hex hex hex
hex ::= [0-9a-fA-F]
)json_brewery";
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_JSON_GRAMMARS_H_

5
tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc
View File

@@ -110,15 +110,16 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
std::uniform_int_distribution<size_t> persona_dist(0, personas.size() - 1); std::uniform_int_distribution<size_t> persona_dist(0, personas.size() - 1);
generated_users_.clear(); generated_users_.clear();
std::unordered_set<std::string> used_email_local_parts;
size_t skipped_count = 0; size_t skipped_count = 0;
size_t export_failed_count = 0; size_t export_failed_count = 0;
const auto generate_record = const auto generate_record =
[this, &rng, &used_email_local_parts, &skipped_count]( [this, &rng, &skipped_count](
const EnrichedCity& city, const UserPersona& persona, const EnrichedCity& city, const UserPersona& persona,
const Name& sampled_name) -> std::optional<UserRecord> { const Name& sampled_name) -> std::optional<UserRecord> {
try { try {
std::unordered_set<std::string> used_email_local_parts;
const UserResult user = const UserResult user =
generator_->GenerateUser(city, persona, sampled_name); generator_->GenerateUser(city, persona, sampled_name);

14
tooling/pipeline/src/data_generation/llama/generate_brewery.cc
View File

@@ -12,6 +12,7 @@
#include <string_view> #include <string_view>
#include <vector> #include <vector>
#include "data_generation/json_grammars.h"
#include "data_generation/llama_generator.h" #include "data_generation/llama_generator.h"
#include "data_generation/llama_generator_helpers.h" #include "data_generation/llama_generator_helpers.h"
@@ -32,19 +33,6 @@ static std::string FormatLocalLanguageCodes(
return formatted; return formatted;
} }
// GBNF grammar for structured brewery JSON output.
// @TODO move to a separate gbnf file if it grows in complexity or is shared
// across modules.
static constexpr std::string_view kBreweryJsonGrammar = R"json_brewery(
root ::= thought-block "{" ws "\"name_en\"" ws ":" ws string ws "," ws "\"description_en\"" ws ":" ws string ws "," ws "\"name_local\"" ws ":" ws string ws "," ws "\"description_local\"" ws ":" ws string ws "}" ws
thought-block ::= [^{]*
ws ::= [ \t\n\r]*
string ::= "\"" char+ "\""
char ::= [^"\\\x7F\x00-\x1F] | [\\] escape
escape ::= ["\\/bfnrt] | "u" hex hex hex hex
hex ::= [0-9a-fA-F]
)json_brewery";
static constexpr int kBreweryInitialMaxTokens = 2800; static constexpr int kBreweryInitialMaxTokens = 2800;
BreweryResult LlamaGenerator::GenerateBrewery( BreweryResult LlamaGenerator::GenerateBrewery(

13
tooling/pipeline/src/data_generation/llama/generate_user.cc
View File

@@ -10,21 +10,10 @@
#include <string> #include <string>
#include <string_view> #include <string_view>
#include "data_generation/json_grammars.h"
#include "data_generation/llama_generator.h" #include "data_generation/llama_generator.h"
#include "data_generation/llama_generator_helpers.h" #include "data_generation/llama_generator_helpers.h"
// GBNF grammar for structured user JSON output.
static constexpr std::string_view kUserJsonGrammar = R"json_user(
root ::= thought-block "{" ws "\"username\"" ws ":" ws string ws "," ws "\"bio\"" ws ":" ws string ws "," ws "\"activity_weight\"" ws ":" ws number ws "}" ws
thought-block ::= [^{]*
ws ::= [ \t\n\r]*
string ::= "\"" char+ "\""
char ::= [^"\\\x7F\x00-\x1F] | [\\] escape
escape ::= ["\\/bfnrt] | "u" hex hex hex hex
hex ::= [0-9a-fA-F]
number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)?
)json_user";
static constexpr int kUserInitialMaxTokens = 1200; static constexpr int kUserInitialMaxTokens = 1200;
UserResult LlamaGenerator::GenerateUser(const EnrichedCity& city, UserResult LlamaGenerator::GenerateUser(const EnrichedCity& city,