From bc8f648a91a2f07b061b3ae1ee1944131c61cda2 Mon Sep 17 00:00:00 2001
From: Ettore Di Giacinto <mudler@users.noreply.github.com>
Date: Mon, 11 Mar 2024 19:49:03 +0100
Subject: [PATCH] fix(doc/examples): set defaults to mirostat (#1820)

The default sampler on some models don't return enough candidates which
leads to a false sense of randomness. Tracing back the code it looks
that with the temperature sampler there might not be enough
candidates to pick from, and since the seed and "randomness" take effect
while picking a good candidate this yields to the same results over and
over.

Fixes https://github.com/mudler/LocalAI/issues/1723 by updating the
examples and documentation to use mirostat instead.
---
 docs/content/docs/advanced/advanced-usage.md  | 7 ++++---
 embedded/models/codellama-7b-gguf.yaml        | 6 +++++-
 embedded/models/dolphin-2.5-mixtral-8x7b.yaml | 5 ++++-
 embedded/models/llava.yaml                    | 3 +++
 embedded/models/mistral-openorca.yaml         | 4 ++++
 embedded/models/mixtral-instruct.yaml         | 4 ++++
 examples/configurations/phi-2.yaml            | 4 ++++
 7 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/docs/content/docs/advanced/advanced-usage.md b/docs/content/docs/advanced/advanced-usage.md
index a892cc369..c9926bab2 100644
--- a/docs/content/docs/advanced/advanced-usage.md
+++ b/docs/content/docs/advanced/advanced-usage.md
@@ -130,13 +130,14 @@ parameters:
   typical_p:
   tfz:
   frequency_penalty:
-  mirostat_eta:
-  mirostat_tau:
-  mirostat: 
+
   rope_freq_base:
   rope_freq_scale:
   negative_prompt_scale:
 
+mirostat_eta:
+mirostat_tau:
+mirostat: 
 # Default context size
 context_size: 512
 # Default number of threads
diff --git a/embedded/models/codellama-7b-gguf.yaml b/embedded/models/codellama-7b-gguf.yaml
index d2a6b5184..413c838b7 100644
--- a/embedded/models/codellama-7b-gguf.yaml
+++ b/embedded/models/codellama-7b-gguf.yaml
@@ -2,10 +2,14 @@ name: codellama-7b-gguf
 backend: transformers
 parameters:
   model: huggingface://TheBloke/CodeLlama-7B-GGUF/codellama-7b.Q4_K_M.gguf
-  temperature: 0.2
+  temperature: 0.5
   top_k: 40
   seed: -1
   top_p: 0.95
+mirostat: 2
+mirostat_eta: 1.0
+mirostat_tau: 1.0
+
 context_size: 4096
 f16: true
 gpu_layers: 90
diff --git a/embedded/models/dolphin-2.5-mixtral-8x7b.yaml b/embedded/models/dolphin-2.5-mixtral-8x7b.yaml
index ba6020c08..12ee1efc2 100644
--- a/embedded/models/dolphin-2.5-mixtral-8x7b.yaml
+++ b/embedded/models/dolphin-2.5-mixtral-8x7b.yaml
@@ -2,10 +2,13 @@ name: dolphin-mixtral-8x7b
 mmap: true
 parameters:
   model: huggingface://TheBloke/dolphin-2.5-mixtral-8x7b-GGUF/dolphin-2.5-mixtral-8x7b.Q2_K.gguf
-  temperature: 0.2
+  temperature: 0.5
   top_k: 40
   top_p: 0.95
   seed: -1
+mirostat: 2
+mirostat_eta: 1.0
+mirostat_tau: 1.0
 template:
   chat_message: |
     <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "user"}}user{{end}}
diff --git a/embedded/models/llava.yaml b/embedded/models/llava.yaml
index 2e571f212..3d240681f 100644
--- a/embedded/models/llava.yaml
+++ b/embedded/models/llava.yaml
@@ -18,6 +18,9 @@ parameters:
   top_k: 40
   top_p: 0.95
   seed: -1
+mirostat: 2
+mirostat_eta: 1.0
+mirostat_tau: 1.0
 
 template:
   chat: |
diff --git a/embedded/models/mistral-openorca.yaml b/embedded/models/mistral-openorca.yaml
index f40d854f7..0794a69b0 100644
--- a/embedded/models/mistral-openorca.yaml
+++ b/embedded/models/mistral-openorca.yaml
@@ -6,6 +6,10 @@ parameters:
   top_k: 40
   top_p: 0.95
   seed: -1
+mirostat: 2
+mirostat_eta: 1.0
+mirostat_tau: 1.0
+
 template:
   chat_message: |
     <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "user"}}user{{end}}
diff --git a/embedded/models/mixtral-instruct.yaml b/embedded/models/mixtral-instruct.yaml
index 3272557a7..246b23240 100644
--- a/embedded/models/mixtral-instruct.yaml
+++ b/embedded/models/mixtral-instruct.yaml
@@ -6,6 +6,10 @@ parameters:
   top_k: 40
   seed: -1
   top_p: 0.95
+mirostat: 2
+mirostat_eta: 1.0
+mirostat_tau: 1.0
+
 template:
   chat: &chat |
     [INST] {{.Input}} [/INST]    
diff --git a/examples/configurations/phi-2.yaml b/examples/configurations/phi-2.yaml
index cac1e9da9..e5a13442d 100644
--- a/examples/configurations/phi-2.yaml
+++ b/examples/configurations/phi-2.yaml
@@ -11,6 +11,10 @@ parameters:
   top_k: 40
   top_p: 0.95
   seed: -1
+  
+mirostat: 2
+mirostat_eta: 1.0
+mirostat_tau: 1.0
 template:
   chat: &template |-
     Instruct: {{.Input}}