Hugging Face's logo

aws-neuron
/
optimum-neuron-cache

Model card Files
xet
Community
621
dacorvo HF Staff commited on
Commit
3263dce
·
1 Parent(s): 804b035

clean-up Trainium 1 cached configurations

Browse files
Files changed (8) hide show
  1. inference-cache-config/trn1/granite.json +66 -0
  2. inference-cache-config/trn1/llama3.json +132 -0
  3. inference-cache-config/trn1/llama4.json +44 -0
  4. inference-cache-config/trn1/mixtral.json +30 -0
  5. inference-cache-config/trn1/phi4.json +30 -0
  6. inference-cache-config/trn1/qwen3-moe.json +28 -0
  7. inference-cache-config/trn1/qwen3.json +104 -0
  8. inference-cache-config/trn1/smollm3.json +22 -0
inference-cache-config/trn1/granite.json ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "ibm-granite/granite-3.1-2b-instruct": [
3
+ {
4
+ "batch_size": 1,
5
+ "sequence_length": 4096,
6
+ "tensor_parallel_size": 2,
7
+ "instance_type" : "trn1"
8
+ },
9
+ {
10
+ "batch_size": 4,
11
+ "sequence_length": 4096,
12
+ "tensor_parallel_size": 2,
13
+ "instance_type" : "trn1"
14
+ },
15
+ {
16
+ "batch_size": 8,
17
+ "sequence_length": 4096,
18
+ "tensor_parallel_size": 2,
19
+ "instance_type" : "trn1"
20
+ },
21
+ {
22
+ "batch_size": 1,
23
+ "sequence_length": 4096,
24
+ "tensor_parallel_size": 8,
25
+ "instance_type" : "trn1"
26
+ },
27
+ {
28
+ "batch_size": 32,
29
+ "sequence_length": 4096,
30
+ "tensor_parallel_size": 8,
31
+ "instance_type" : "trn1"
32
+ }
33
+ ],
34
+ "ibm-granite/granite-3.1-8b-instruct": [
35
+ {
36
+ "batch_size": 1,
37
+ "sequence_length": 4096,
38
+ "tensor_parallel_size": 2,
39
+ "instance_type" : "trn1"
40
+ },
41
+ {
42
+ "batch_size": 4,
43
+ "sequence_length": 4096,
44
+ "tensor_parallel_size": 2,
45
+ "instance_type" : "trn1"
46
+ },
47
+ {
48
+ "batch_size": 8,
49
+ "sequence_length": 4096,
50
+ "tensor_parallel_size": 2,
51
+ "instance_type" : "trn1"
52
+ },
53
+ {
54
+ "batch_size": 1,
55
+ "sequence_length": 4096,
56
+ "tensor_parallel_size": 8,
57
+ "instance_type" : "trn1"
58
+ },
59
+ {
60
+ "batch_size": 32,
61
+ "sequence_length": 4096,
62
+ "tensor_parallel_size": 8,
63
+ "instance_type" : "trn1"
64
+ }
65
+ ]
66
+ }
inference-cache-config/trn1/llama3.json ADDED
@@ -0,0 +1,132 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "meta-llama/Meta-Llama-3.1-8B": [
3
+ {
4
+ "batch_size": 1,
5
+ "sequence_length": 4096,
6
+ "tensor_parallel_size": 2,
7
+ "instance_type" : "trn1"
8
+ },
9
+ {
10
+ "batch_size": 4,
11
+ "sequence_length": 4096,
12
+ "tensor_parallel_size": 2,
13
+ "instance_type" : "trn1"
14
+ },
15
+ {
16
+ "batch_size": 8,
17
+ "sequence_length": 4096,
18
+ "tensor_parallel_size": 2,
19
+ "instance_type" : "trn1"
20
+ },
21
+ {
22
+ "batch_size": 4,
23
+ "sequence_length": 4096,
24
+ "tensor_parallel_size": 8,
25
+ "instance_type" : "trn1"
26
+ },
27
+ {
28
+ "batch_size": 8,
29
+ "sequence_length": 4096,
30
+ "tensor_parallel_size": 8,
31
+ "instance_type" : "trn1"
32
+ },
33
+ {
34
+ "batch_size": 16,
35
+ "sequence_length": 4096,
36
+ "tensor_parallel_size": 8,
37
+ "instance_type" : "trn1"
38
+ },
39
+ {
40
+ "batch_size": 32,
41
+ "sequence_length": 4096,
42
+ "tensor_parallel_size": 8,
43
+ "instance_type" : "trn1"
44
+ }
45
+ ],
46
+ "meta-llama/Llama-2-7b-hf": [
47
+ {
48
+ "batch_size": 1,
49
+ "sequence_length": 2048,
50
+ "tensor_parallel_size": 2,
51
+ "instance_type" : "trn1"
52
+ }
53
+ ],
54
+ "meta-llama/Llama-2-13b-hf": [
55
+ {
56
+ "batch_size": 1,
57
+ "sequence_length": 2048,
58
+ "tensor_parallel_size": 8,
59
+ "instance_type" : "trn1"
60
+ }
61
+ ],
62
+ "meta-llama/Meta-Llama-3-8B": [
63
+ {
64
+ "batch_size": 1,
65
+ "sequence_length": 4096,
66
+ "tensor_parallel_size": 2,
67
+ "instance_type" : "trn1"
68
+ }
69
+ ],
70
+ "meta-llama/Llama-3.2-1B": [
71
+ {
72
+ "batch_size": 1,
73
+ "sequence_length": 4096,
74
+ "tensor_parallel_size": 2,
75
+ "instance_type" : "trn1"
76
+ },
77
+ {
78
+ "batch_size": 4,
79
+ "sequence_length": 4096,
80
+ "tensor_parallel_size": 2,
81
+ "instance_type" : "trn1"
82
+ }
83
+ ],
84
+ "meta-llama/Llama-3.2-3B": [
85
+ {
86
+ "batch_size": 1,
87
+ "sequence_length": 4096,
88
+ "tensor_parallel_size": 2,
89
+ "instance_type" : "trn1"
90
+ },
91
+ {
92
+ "batch_size": 4,
93
+ "sequence_length": 4096,
94
+ "tensor_parallel_size": 2,
95
+ "instance_type" : "trn1"
96
+ }
97
+ ],
98
+ "TinyLlama/TinyLlama-1.1B-Chat-v1.0": [
99
+ {
100
+ "batch_size": 1,
101
+ "sequence_length": 2048,
102
+ "tensor_parallel_size": 2,
103
+ "instance_type" : "trn1"
104
+ }
105
+ ],
106
+ "meta-llama/Llama-3.3-70B-Instruct": [
107
+ {
108
+ "batch_size": 1,
109
+ "sequence_length": 4096,
110
+ "tensor_parallel_size": 24,
111
+ "instance_type" : "trn1"
112
+ },
113
+ {
114
+ "batch_size": 8,
115
+ "sequence_length": 4096,
116
+ "tensor_parallel_size": 24,
117
+ "instance_type" : "trn1"
118
+ },
119
+ {
120
+ "batch_size": 1,
121
+ "sequence_length": 4096,
122
+ "tensor_parallel_size": 32,
123
+ "instance_type" : "trn1"
124
+ },
125
+ {
126
+ "batch_size": 8,
127
+ "sequence_length": 4096,
128
+ "tensor_parallel_size": 32,
129
+ "instance_type" : "trn1"
130
+ }
131
+ ]
132
+ }
inference-cache-config/trn1/llama4.json ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "meta-llama/Llama-4-Scout-17B-16E-Instruct": [
3
+ {
4
+ "task": "text-generation",
5
+ "instance_type": "trn1",
6
+ "batch_size": 1,
7
+ "sequence_length": 4096,
8
+ "tensor_parallel_size": 16,
9
+ "auto_cast_type": "bf16"
10
+ },
11
+ {
12
+ "task": "text-generation",
13
+ "instance_type": "trn1",
14
+ "batch_size": 4,
15
+ "sequence_length": 4096,
16
+ "tensor_parallel_size": 16,
17
+ "auto_cast_type": "bf16"
18
+ },
19
+ {
20
+ "task": "text-generation",
21
+ "instance_type": "trn1",
22
+ "batch_size": 1,
23
+ "sequence_length": 4096,
24
+ "tensor_parallel_size": 32,
25
+ "auto_cast_type": "bf16"
26
+ },
27
+ {
28
+ "task": "text-generation",
29
+ "instance_type": "trn1",
30
+ "batch_size": 4,
31
+ "sequence_length": 4096,
32
+ "tensor_parallel_size": 32,
33
+ "auto_cast_type": "bf16"
34
+ },
35
+ {
36
+ "task": "text-generation",
37
+ "instance_type": "trn1",
38
+ "batch_size": 8,
39
+ "sequence_length": 4096,
40
+ "tensor_parallel_size": 32,
41
+ "auto_cast_type": "bf16"
42
+ }
43
+ ]
44
+ }
inference-cache-config/trn1/mixtral.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "mistralai/Mixtral-8x7B-Instruct-v0.1": [
3
+ {
4
+ "batch_size": 1,
5
+ "sequence_length": 4096,
6
+ "tensor_parallel_size": 24,
7
+ "instance_type" : "trn1"
8
+ },
9
+ {
10
+ "batch_size": 4,
11
+ "sequence_length": 4096,
12
+ "tensor_parallel_size": 24,
13
+ "instance_type" : "trn1"
14
+ }
15
+ ],
16
+ "mistralai/Mixtral-8x22B-Instruct-v0.1": [
17
+ {
18
+ "batch_size": 1,
19
+ "sequence_length": 4096,
20
+ "tensor_parallel_size": 24,
21
+ "instance_type" : "trn1"
22
+ },
23
+ {
24
+ "batch_size": 4,
25
+ "sequence_length": 4096,
26
+ "tensor_parallel_size": 24,
27
+ "instance_type" : "trn1"
28
+ }
29
+ ]
30
+ }
inference-cache-config/trn1/phi4.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "microsoft/Phi-3-mini-4k-instruct": [
3
+ {
4
+ "batch_size": 1,
5
+ "sequence_length": 4096,
6
+ "tensor_parallel_size": 2,
7
+ "instance_type" : "trn1"
8
+ },
9
+ {
10
+ "batch_size": 4,
11
+ "sequence_length": 4096,
12
+ "tensor_parallel_size": 2,
13
+ "instance_type" : "trn1"
14
+ }
15
+ ],
16
+ "microsoft/phi-4": [
17
+ {
18
+ "batch_size": 1,
19
+ "sequence_length": 4096,
20
+ "tensor_parallel_size": 10,
21
+ "instance_type" : "trn1"
22
+ },
23
+ {
24
+ "batch_size": 16,
25
+ "sequence_length": 4096,
26
+ "tensor_parallel_size": 10,
27
+ "instance_type" : "trn1"
28
+ }
29
+ ]
30
+ }
inference-cache-config/trn1/qwen3-moe.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "Qwen/Qwen3-30B-A3B-Instruct-2507": [
3
+ {
4
+ "batch_size": 1,
5
+ "sequence_length": 4096,
6
+ "tensor_parallel_size": 8,
7
+ "instance_type" : "trn1"
8
+ },
9
+ {
10
+ "batch_size": 4,
11
+ "sequence_length": 4096,
12
+ "tensor_parallel_size": 8,
13
+ "instance_type" : "trn1"
14
+ },
15
+ {
16
+ "batch_size": 8,
17
+ "sequence_length": 4096,
18
+ "tensor_parallel_size": 16,
19
+ "instance_type" : "trn1"
20
+ },
21
+ {
22
+ "batch_size": 8,
23
+ "sequence_length": 4096,
24
+ "tensor_parallel_size": 32,
25
+ "instance_type" : "trn1"
26
+ }
27
+ ]
28
+ }
inference-cache-config/trn1/qwen3.json ADDED
@@ -0,0 +1,104 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "Qwen/Qwen3-8B": [
3
+ {
4
+ "batch_size": 1,
5
+ "sequence_length": 4096,
6
+ "tensor_parallel_size": 2,
7
+ "instance_type" : "trn1"
8
+ },
9
+ {
10
+ "batch_size": 4,
11
+ "sequence_length": 4096,
12
+ "tensor_parallel_size": 2,
13
+ "instance_type" : "trn1"
14
+ },
15
+ {
16
+ "batch_size": 8,
17
+ "sequence_length": 4096,
18
+ "tensor_parallel_size": 2,
19
+ "instance_type" : "trn1"
20
+ },
21
+ {
22
+ "batch_size": 1,
23
+ "sequence_length": 4096,
24
+ "tensor_parallel_size": 8,
25
+ "instance_type" : "trn1"
26
+ },
27
+ {
28
+ "batch_size": 32,
29
+ "sequence_length": 4096,
30
+ "tensor_parallel_size": 8,
31
+ "instance_type" : "trn1"
32
+ }
33
+ ],
34
+ "Qwen/Qwen3-0.6B": [
35
+ {
36
+ "batch_size": 1,
37
+ "sequence_length": 4096,
38
+ "tensor_parallel_size": 2,
39
+ "instance_type" : "trn1"
40
+ },
41
+ {
42
+ "batch_size": 4,
43
+ "sequence_length": 4096,
44
+ "tensor_parallel_size": 2,
45
+ "instance_type" : "trn1"
46
+ }
47
+ ],
48
+ "Qwen/Qwen3-1.7B": [
49
+ {
50
+ "batch_size": 1,
51
+ "sequence_length": 4096,
52
+ "tensor_parallel_size": 2,
53
+ "instance_type" : "trn1"
54
+ },
55
+ {
56
+ "batch_size": 4,
57
+ "sequence_length": 4096,
58
+ "tensor_parallel_size": 2,
59
+ "instance_type" : "trn1"
60
+ }
61
+ ],
62
+ "Qwen/Qwen3-4B": [
63
+ {
64
+ "batch_size": 1,
65
+ "sequence_length": 4096,
66
+ "tensor_parallel_size": 2,
67
+ "instance_type" : "trn1"
68
+ },
69
+ {
70
+ "batch_size": 4,
71
+ "sequence_length": 4096,
72
+ "tensor_parallel_size": 2,
73
+ "instance_type" : "trn1"
74
+ }
75
+ ],
76
+ "Qwen/Qwen3-14B": [
77
+ {
78
+ "batch_size": 1,
79
+ "sequence_length": 4096,
80
+ "tensor_parallel_size": 8,
81
+ "instance_type" : "trn1"
82
+ },
83
+ {
84
+ "batch_size": 16,
85
+ "sequence_length": 4096,
86
+ "tensor_parallel_size": 8,
87
+ "instance_type" : "trn1"
88
+ }
89
+ ],
90
+ "Qwen/Qwen3-32B": [
91
+ {
92
+ "batch_size": 1,
93
+ "sequence_length": 4096,
94
+ "tensor_parallel_size": 8,
95
+ "instance_type" : "trn1"
96
+ },
97
+ {
98
+ "batch_size": 16,
99
+ "sequence_length": 4096,
100
+ "tensor_parallel_size": 8,
101
+ "instance_type" : "trn1"
102
+ }
103
+ ]
104
+ }
inference-cache-config/trn1/smollm3.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "HuggingFaceTB/SmolLM3-3B": [
3
+ {
4
+ "batch_size": 1,
5
+ "sequence_length": 4096,
6
+ "tensor_parallel_size": 2,
7
+ "instance_type" : "trn1"
8
+ },
9
+ {
10
+ "batch_size": 4,
11
+ "sequence_length": 4096,
12
+ "tensor_parallel_size": 2,
13
+ "instance_type" : "trn1"
14
+ },
15
+ {
16
+ "batch_size": 16,
17
+ "sequence_length": 4096,
18
+ "tensor_parallel_size": 2,
19
+ "instance_type" : "trn1"
20
+ }
21
+ ]
22
+ }