import GboxSDK from "gbox-sdk";
const gboxSDK = new GboxSDK({
apiKey: process.env["GBOX_API_KEY"] // This is the default and can be omitted
});
async function main() {
const box = await gboxSDK.create({ type: "android" });
await box.action.ai("click the login button");
await box.action.ai({
instruction: "click the login button",
background: "You are a QA engineer. You are testing the login page of the application."
});
}
main();
{
"message": "Action executed successfully",
"screenshot": {
"trace": {
"uri": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA..."
},
"before": {
"uri": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA..."
},
"after": {
"uri": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA..."
}
},
"output": "Action executed successfully",
"aiResponse": {
"model": "open-ai-computer-use-preview",
"reasoning": "The button is on the login page. so i will click the button",
"messages": [
"The button is on the login page. so i will click the button",
"The button is clicked"
],
"actions": [
{
"type": "click",
"x": 100,
"y": 100
}
]
}
}
Use natural language instructions to perform UI operations on the box. The endpoint will stream progress events before and after the action is executed. If you don’t need intermediate events, set stream to false.
import GboxSDK from "gbox-sdk";
const gboxSDK = new GboxSDK({
apiKey: process.env["GBOX_API_KEY"] // This is the default and can be omitted
});
async function main() {
const box = await gboxSDK.create({ type: "android" });
await box.action.ai("click the login button");
await box.action.ai({
instruction: "click the login button",
background: "You are a QA engineer. You are testing the login page of the application."
});
}
main();
{
"message": "Action executed successfully",
"screenshot": {
"trace": {
"uri": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA..."
},
"before": {
"uri": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA..."
},
"after": {
"uri": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA..."
}
},
"output": "Action executed successfully",
"aiResponse": {
"model": "open-ai-computer-use-preview",
"reasoning": "The button is on the login page. so i will click the button",
"messages": [
"The button is on the login page. so i will click the button",
"The button is clicked"
],
"actions": [
{
"type": "click",
"x": 100,
"y": 100
}
]
}
}
Enter your API Key in the format: Bearer <token>. Get it from https://gbox.ai
Box ID
"c9bdc193-b54b-4ddb-a035-5ac0c598d32d"
AI action configuration
Result of AI action execution with screenshot
Was this page helpful?