AaronPo/the-biergarten-app
1
0
Fork 0
mirror of https://github.com/aaronpo97/the-biergarten-app.git synced 2026-07-16 09:37:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(pipeline): restructure config, add PromptDirectory, consolidate SQLite layer (#217)

Browse Source 
* Refactor ApplicationOptions to separate config concerns

* add prompt dir app option

* readability updates: remove magic numbers, update comments

* codebase formatting

* Update docs

* Extract argument parsing, timer out of
This commit is contained in:
Aaron Po
2026-05-02 18:27:14 -04:00
committed by GitHub
parent 641a479b6a
commit b1dc8e0b5d
35 changed files with 561 additions and 310 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
tooling/pipeline/src/data_generation/llama/load.cc
View File

@@ -14,6 +14,10 @@
#include "data_generation/llama_generator.h"
#include "llama.h"
// Maximum batch size for decode operations. Capping the batch prevents
// excessive memory allocation while maintaining inference performance.
static constexpr uint32_t kMaxBatchSize = 5000U;
void LlamaGenerator::Load(const std::string& model_path) {
context_.reset();
model_.reset();
@@ -28,7 +32,7 @@ void LlamaGenerator::Load(const std::string& model_path) {
llama_context_params context_params = llama_context_default_params();
context_params.n_ctx = n_ctx_;
context_params.n_batch = std::min(n_ctx_, static_cast<uint32_t>(5000));
context_params.n_batch = std::min(n_ctx_, kMaxBatchSize);
LlamaGenerator::ContextHandle loaded_context(
llama_init_from_model(loaded_model.get(), context_params));