ray/examples/config/ray.sub1b.drafter.json at main · razroo/ray · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
{
  "profile": "sub1b",
  "model": {
    "maxOutputTokens": 180,
    "operational": {
      "recommendedPromptFormat": "native-template",
      "supportsJsonMode": true,
      "tokensPerSecondTarget": 16,
      "memoryClassMiB": 4096,
      "preferredCtxSize": 3072,
      "chatTemplateKnown": true
    },
    "adapter": {
      "kind": "llama.cpp",
      "timeoutMs": 18000,
      "slotSnapshotTimeoutMs": 250,
      "promptScaffoldCacheEntries": 384,
      "warmupRequests": [
        {
          "templateId": "email.cold_outreach.v1",
          "templateVariables": {
            "recipientRole": "VP Revenue Operations",
            "topic": "faster follow-up drafting",
            "valueProp": "turn CRM context into first-pass replies",
            "companyContext": "B2B SaaS team"
          },
          "maxTokens": 120,
          "seed": 21
        },
        {
          "templateId": "email.reply_rewrite.v1",
          "templateVariables": {
            "rewriteGoal": "Sound clear and helpful.",
            "replyText": "I can take a look next week if you send details."
          },
          "maxTokens": 100,
          "seed": 22
        }
      ]
    }
  },
  "scheduler": {
    "concurrency": 2,
    "maxQueue": 72,
    "maxQueuedTokens": 22000,
    "maxInflightTokens": 4096,
    "requestTimeoutMs": 20000,
    "dedupeInflight": true,
    "batchWindowMs": 10,
    "affinityLookahead": 24,
    "shortJobMaxTokens": 96
  },
  "gracefulDegradation": {
    "queueDepthThreshold": 14,
    "maxPromptChars": 5500,
    "degradeToMaxTokens": 112
  },
  "tags": {
    "target": "sub1b",
    "modelRole": "drafter",
    "modelSize": "0.5b-class",
    "hardware": "hetzner-cx23-class"
  }
}