Compare commits

...

2 Commits

Author SHA1 Message Date
wbpxre150
36b4f7e064 llama : print timings on ctrl+c exit (#1021)
* print timings on ctrl+c exit

* remove redundant free memory call.

* add global pointer to ctx.
2023-04-22 11:56:35 +03:00
eiery
10f19c1121 llama : have n_batch default to 512 (#1091)
* set default n_batch to 512 when using BLAS

* spacing

* alternate implementation of setting different n_batch for BLAS

* set n_batch to 512 for all cases
2023-04-22 11:27:05 +03:00
2 changed files with 5 additions and 2 deletions

View File

@@ -20,7 +20,7 @@ struct gpt_params {
int32_t repeat_last_n = 64; // last n tokens to penalize
int32_t n_parts = -1; // amount of model parts (-1 = determine from model dimensions)
int32_t n_ctx = 512; // context size
int32_t n_batch = 8; // batch size for prompt processing
int32_t n_batch = 512; // batch size for prompt processing (must be >=32 to use BLAS)
int32_t n_keep = 0; // number of tokens to keep from initial prompt
// sampling parameters

View File

@@ -25,6 +25,7 @@
#endif
static console_state con_st;
static llama_context ** g_ctx;
static bool is_interacting = false;
@@ -36,6 +37,7 @@ void sigint_handler(int signo) {
if (!is_interacting) {
is_interacting=true;
} else {
llama_print_timings(*g_ctx);
_exit(130);
}
}
@@ -92,8 +94,9 @@ int main(int argc, char ** argv) {
// params.prompt = R"(// this function checks if the number n is prime
//bool is_prime(int n) {)";
llama_context * ctx;
g_ctx = &ctx;
// load the model
{