Readme
This model doesn't have a readme.
andreasjansson
/
llama-2-13b-gguf
Llama-2 13B with support for grammars and jsonschema
Input
Run this model in Node.js with one line of code:
npx create-replicate --model=andreasjansson/llama-2-13b-gguf
or set up a project from scratch
Install Replicate’s Node.js client library:
npm install replicateSet the
REPLICATE_API_TOKEN environment variable:export REPLICATE_API_TOKEN=<paste-your-token-here>Find your API token in your account settings.
Import and set up the client:
import Replicate from "replicate";
const replicate = new Replicate({
auth: process.env.REPLICATE_API_TOKEN,
});Run andreasjansson/llama-2-13b-gguf using Replicate’s API. Check out the model's schema for an overview of inputs and outputs.
const output = await replicate.run(
"andreasjansson/llama-2-13b-gguf:b988dba0de3af274d9d913eaf277debab37602251b8315eccbfddf9c29d82983",
{
input: {
prompt: "Send a email from andreas86@telia.se to myfriend@telia.se where you discuss the weather. In the body, describe the current weather in Stockholm as detailed as possible.\n\nRespond with json that adheres to the following jsonschema:\n\n{jsonschema}",
grammar: "",
max_tokens: 500
}
}
);
console.log(output);To learn more, take a look at the guide on getting started with Node.js.
Install Replicate’s Python client library:
pip install replicateSet the
REPLICATE_API_TOKEN environment variable:export REPLICATE_API_TOKEN=<paste-your-token-here>Find your API token in your account settings.
Import the client:
import replicateRun andreasjansson/llama-2-13b-gguf using Replicate’s API. Check out the model's schema for an overview of inputs and outputs.
output = replicate.run(
"andreasjansson/llama-2-13b-gguf:b988dba0de3af274d9d913eaf277debab37602251b8315eccbfddf9c29d82983",
input={
"prompt": "Send a email from andreas86@telia.se to myfriend@telia.se where you discuss the weather. In the body, describe the current weather in Stockholm as detailed as possible.\n\nRespond with json that adheres to the following jsonschema:\n\n{jsonschema}",
"grammar": "",
"max_tokens": 500
}
)
# The andreasjansson/llama-2-13b-gguf model can stream output as it's running.
# The predict method returns an iterator, and you can iterate over that output.
for item in output:
# https://replicate.com/andreasjansson/llama-2-13b-gguf/api#output-schema
print(item, end="")To learn more, take a look at the guide on getting started with Python.
Set the
REPLICATE_API_TOKEN environment variable:export REPLICATE_API_TOKEN=<paste-your-token-here>Find your API token in your account settings.
Run andreasjansson/llama-2-13b-gguf using Replicate’s API. Check out the model's schema for an overview of inputs and outputs.
curl -s -X POST \
-H "Authorization: Bearer $REPLICATE_API_TOKEN" \
-H "Content-Type: application/json" \
-H "Prefer: wait" \
-d $'{
"version": "andreasjansson/llama-2-13b-gguf:b988dba0de3af274d9d913eaf277debab37602251b8315eccbfddf9c29d82983",
"input": {
"prompt": "Send a email from andreas86@telia.se to myfriend@telia.se where you discuss the weather. In the body, describe the current weather in Stockholm as detailed as possible.\\n\\nRespond with json that adheres to the following jsonschema:\\n\\n{jsonschema}",
"grammar": "",
"max_tokens": 500
}
}' \
https://api.replicate.com/v1/predictionsTo learn more, take a look at Replicate’s HTTP API reference docs.
Add a payment method to run this model.
Sign in with GitHub
By signing in, you agree to our
terms of service and privacy policy
Output
{"bcc":[],"body":"Hello, today it's rainy and 14 degrees.","cc":[],"from":"andreas86@telia.se","subject":"Today weather in Stockholm","to":["myfriend@telia.se"]}
{
"completed_at": "2023-09-22T22:14:09.027559Z",
"created_at": "2023-09-22T22:14:03.891094Z",
"data_removed": false,
"error": null,
"id": "bvxsf7dbip4debhrndy7s5ijou",
"input": {
"top_k": 10,
"top_p": 0.95,
"prompt": "Send a email from andreas86@telia.se to myfriend@telia.se where you discuss the weather. In the body, describe the current weather in Stockholm as detailed as possible.\n\nRespond with json that adheres to the following jsonschema:\n\n{jsonschema}",
"grammar": "",
"jsonschema": "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"type\": \"object\",\n \"properties\": {\n \"from\": {\n \"type\": \"string\",\n \"format\": \"email\",\n \"description\": \"The email address of the sender.\"\n },\n \"to\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\",\n \"format\": \"email\"\n },\n \"description\": \"An array of email addresses to send the email to.\"\n },\n \"cc\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\",\n \"format\": \"email\"\n },\n \"description\": \"An array of email addresses to be included in the CC field.\"\n },\n \"bcc\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\",\n \"format\": \"email\"\n },\n \"description\": \"An array of email addresses to be included in the BCC field.\"\n },\n \"subject\": {\n \"type\": \"string\",\n \"description\": \"The subject of the email.\"\n },\n \"body\": {\n \"type\": \"string\",\n \"description\": \"The body text of the email.\"\n }\n },\n \"required\": [\"from\", \"to\", \"subject\", \"body\"],\n \"additionalProperties\": false\n}",
"max_tokens": 500,
"temperature": 0.8,
"mirostat_mode": "Disabled",
"repeat_penalty": 1.1,
"mirostat_entropy": 5,
"presence_penalty": 0,
"frequency_penalty": 0,
"mirostat_learning_rate": 0.1
},
"logs": "from_string grammar:\nspace ::= space_1\nspace_1 ::= [ ] |\nstring ::= [\"] string_5 [\"] space\nstring_3 ::= [^\"\\] | [\\] string_4\nstring_4 ::= [\"\\/bfnrt] | [u] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]\nstring_5 ::= string_3 string_5 |\nbcc ::= [[] space bcc_10 []] space\nbcc_7 ::= string bcc_9\nbcc_8 ::= [,] space string\nbcc_9 ::= bcc_8 bcc_9 |\nbcc_10 ::= bcc_7 |\ncc ::= [[] space cc_15 []] space\ncc_12 ::= string cc_14\ncc_13 ::= [,] space string\ncc_14 ::= cc_13 cc_14 |\ncc_15 ::= cc_12 |\nto ::= [[] space to_20 []] space\nto_17 ::= string to_19\nto_18 ::= [,] space string\nto_19 ::= to_18 to_19 |\nto_20 ::= to_17 |\nroot ::= [{] space [\"] [b] [c] [c] [\"] space [:] space bcc [,] space [\"] [b] [o] [d] [y] [\"] space [:] space string [,] space [\"] [c] [c] [\"] space [:] space cc [,] space [\"] [f] [r] [o] [m] [\"] space [:] space string [,] space [\"] [s] [u] [b] [j] [e] [c] [t] [\"] space [:] space string [,] space [\"] [t] [o] [\"] space [:] space to [}] space\nPrompt:\nSend a email from andreas86@telia.se to myfriend@telia.se where you discuss the weather. In the body, describe the current weather in Stockholm as detailed as possible.\nRespond with json that adheres to the following jsonschema:\n{\n\"$schema\": \"http://json-schema.org/draft-07/schema#\",\n\"type\": \"object\",\n\"properties\": {\n\"from\": {\n\"type\": \"string\",\n\"format\": \"email\",\n\"description\": \"The email address of the sender.\"\n},\n\"to\": {\n\"type\": \"array\",\n\"items\": {\n\"type\": \"string\",\n\"format\": \"email\"\n},\n\"description\": \"An array of email addresses to send the email to.\"\n},\n\"cc\": {\n\"type\": \"array\",\n\"items\": {\n\"type\": \"string\",\n\"format\": \"email\"\n},\n\"description\": \"An array of email addresses to be included in the CC field.\"\n},\n\"bcc\": {\n\"type\": \"array\",\n\"items\": {\n\"type\": \"string\",\n\"format\": \"email\"\n},\n\"description\": \"An array of email addresses to be included in the BCC field.\"\n},\n\"subject\": {\n\"type\": \"string\",\n\"description\": \"The subject of the email.\"\n},\n\"body\": {\n\"type\": \"string\",\n\"description\": \"The body text of the email.\"\n}\n},\n\"required\": [\"from\", \"to\", \"subject\", \"body\"],\n\"additionalProperties\": false\n}\nllama_print_timings: load time = 576.68 ms\nllama_print_timings: sample time = 518.20 ms / 63 runs ( 8.23 ms per token, 121.57 tokens per second)\nllama_print_timings: prompt eval time = 576.15 ms / 411 tokens ( 1.40 ms per token, 713.36 tokens per second)\nllama_print_timings: eval time = 1207.52 ms / 62 runs ( 19.48 ms per token, 51.34 tokens per second)\nllama_print_timings: total time = 2441.55 ms",
"metrics": {
"predict_time": 5.121619,
"total_time": 5.136465
},
"output": [
"{\"",
"b",
"cc",
"\":",
"[",
"],",
"\"",
"body",
"\":\"",
"Hello",
",",
" today",
" it",
"'",
"s",
" ra",
"iny",
" and",
" ",
"1",
"4",
" degrees",
".",
"\",\"",
"cc",
"\":",
"[",
"],",
"\"",
"from",
"\":\"",
"and",
"reas",
"8",
"6",
"@",
"t",
"elia",
".",
"se",
"\",\"",
"subject",
"\":\"",
"T",
"od",
"ay",
" weather",
" in",
" Stockholm",
"\",\"",
"to",
"\":",
"[\"",
"my",
"friend",
"@",
"t",
"elia",
".",
"se",
"\"]",
"}"
],
"started_at": "2023-09-22T22:14:03.905940Z",
"status": "succeeded",
"urls": {
"get": "https://api.replicate.com/v1/predictions/bvxsf7dbip4debhrndy7s5ijou",
"cancel": "https://api.replicate.com/v1/predictions/bvxsf7dbip4debhrndy7s5ijou/cancel"
},
"version": "19eb0a04b1f88bd2ead5e66cdb70eb25edfffdf89231eeb8a9f2e1b19464d337"
}Generated in
from_string grammar:
space ::= space_1
space_1 ::= [ ] |
string ::= ["] string_5 ["] space
string_3 ::= [^"\] | [\] string_4
string_4 ::= ["\/bfnrt] | [u] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]
string_5 ::= string_3 string_5 |
bcc ::= [[] space bcc_10 []] space
bcc_7 ::= string bcc_9
bcc_8 ::= [,] space string
bcc_9 ::= bcc_8 bcc_9 |
bcc_10 ::= bcc_7 |
cc ::= [[] space cc_15 []] space
cc_12 ::= string cc_14
cc_13 ::= [,] space string
cc_14 ::= cc_13 cc_14 |
cc_15 ::= cc_12 |
to ::= [[] space to_20 []] space
to_17 ::= string to_19
to_18 ::= [,] space string
to_19 ::= to_18 to_19 |
to_20 ::= to_17 |
root ::= [{] space ["] [b] [c] [c] ["] space [:] space bcc [,] space ["] [b] [o] [d] [y] ["] space [:] space string [,] space ["] [c] [c] ["] space [:] space cc [,] space ["] [f] [r] [o] [m] ["] space [:] space string [,] space ["] [s] [u] [b] [j] [e] [c] [t] ["] space [:] space string [,] space ["] [t] [o] ["] space [:] space to [}] space
Prompt:
Send a email from andreas86@telia.se to myfriend@telia.se where you discuss the weather. In the body, describe the current weather in Stockholm as detailed as possible.
Respond with json that adheres to the following jsonschema:
{
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
"properties": {
"from": {
"type": "string",
"format": "email",
"description": "The email address of the sender."
},
"to": {
"type": "array",
"items": {
"type": "string",
"format": "email"
},
"description": "An array of email addresses to send the email to."
},
"cc": {
"type": "array",
"items": {
"type": "string",
"format": "email"
},
"description": "An array of email addresses to be included in the CC field."
},
"bcc": {
"type": "array",
"items": {
"type": "string",
"format": "email"
},
"description": "An array of email addresses to be included in the BCC field."
},
"subject": {
"type": "string",
"description": "The subject of the email."
},
"body": {
"type": "string",
"description": "The body text of the email."
}
},
"required": ["from", "to", "subject", "body"],
"additionalProperties": false
}
llama_print_timings: load time = 576.68 ms
llama_print_timings: sample time = 518.20 ms / 63 runs ( 8.23 ms per token, 121.57 tokens per second)
llama_print_timings: prompt eval time = 576.15 ms / 411 tokens ( 1.40 ms per token, 713.36 tokens per second)
llama_print_timings: eval time = 1207.52 ms / 62 runs ( 19.48 ms per token, 51.34 tokens per second)
llama_print_timings: total time = 2441.55 msThis output was created using a different version of the model, andreasjansson/llama-2-13b-gguf:19eb0a04.
Run time and cost
This model costs approximately $0.015 to run on Replicate, or 66 runs per $1, but this varies depending on your inputs. It is also open source and you can run it on your own computer with Docker.
This model runs on Nvidia L40S GPU hardware. Predictions typically complete within 16 seconds. The predict time for this model varies significantly based on the inputs.