Process Dataset
curl --request POST \
--url https://preprocessing-service-433968519479.us-central1.run.app/datasets/process \
--header 'Content-Type: application/json' \
--data '
{
"dataset_name": "<string>",
"dataset_id": "<string>",
"config": {
"field_mappings": {},
"normalize_whitespace": true,
"augmentation_config": {
"augmentation_factor": 1.5,
"use_eda": false,
"use_back_translation": false,
"use_paraphrasing": false,
"use_synthesis": false,
"gemini_api_key": "<string>",
"synthesis_ratio": 123,
"custom_prompt": "<string>"
},
"split_config": {
"train_split": "<string>",
"test_split": "<string>",
"type": "hf_split"
}
},
"dataset_subset": "default"
}
'import requests
url = "https://preprocessing-service-433968519479.us-central1.run.app/datasets/process"
payload = {
"dataset_name": "<string>",
"dataset_id": "<string>",
"config": {
"field_mappings": {},
"normalize_whitespace": True,
"augmentation_config": {
"augmentation_factor": 1.5,
"use_eda": False,
"use_back_translation": False,
"use_paraphrasing": False,
"use_synthesis": False,
"gemini_api_key": "<string>",
"synthesis_ratio": 123,
"custom_prompt": "<string>"
},
"split_config": {
"train_split": "<string>",
"test_split": "<string>",
"type": "hf_split"
}
},
"dataset_subset": "default"
}
headers = {"Content-Type": "application/json"}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({
dataset_name: '<string>',
dataset_id: '<string>',
config: {
field_mappings: {},
normalize_whitespace: true,
augmentation_config: {
augmentation_factor: 1.5,
use_eda: false,
use_back_translation: false,
use_paraphrasing: false,
use_synthesis: false,
gemini_api_key: '<string>',
synthesis_ratio: 123,
custom_prompt: '<string>'
},
split_config: {train_split: '<string>', test_split: '<string>', type: 'hf_split'}
},
dataset_subset: 'default'
})
};
fetch('https://preprocessing-service-433968519479.us-central1.run.app/datasets/process', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://preprocessing-service-433968519479.us-central1.run.app/datasets/process",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'dataset_name' => '<string>',
'dataset_id' => '<string>',
'config' => [
'field_mappings' => [
],
'normalize_whitespace' => true,
'augmentation_config' => [
'augmentation_factor' => 1.5,
'use_eda' => false,
'use_back_translation' => false,
'use_paraphrasing' => false,
'use_synthesis' => false,
'gemini_api_key' => '<string>',
'synthesis_ratio' => 123,
'custom_prompt' => '<string>'
],
'split_config' => [
'train_split' => '<string>',
'test_split' => '<string>',
'type' => 'hf_split'
]
],
'dataset_subset' => 'default'
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://preprocessing-service-433968519479.us-central1.run.app/datasets/process"
payload := strings.NewReader("{\n \"dataset_name\": \"<string>\",\n \"dataset_id\": \"<string>\",\n \"config\": {\n \"field_mappings\": {},\n \"normalize_whitespace\": true,\n \"augmentation_config\": {\n \"augmentation_factor\": 1.5,\n \"use_eda\": false,\n \"use_back_translation\": false,\n \"use_paraphrasing\": false,\n \"use_synthesis\": false,\n \"gemini_api_key\": \"<string>\",\n \"synthesis_ratio\": 123,\n \"custom_prompt\": \"<string>\"\n },\n \"split_config\": {\n \"train_split\": \"<string>\",\n \"test_split\": \"<string>\",\n \"type\": \"hf_split\"\n }\n },\n \"dataset_subset\": \"default\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://preprocessing-service-433968519479.us-central1.run.app/datasets/process")
.header("Content-Type", "application/json")
.body("{\n \"dataset_name\": \"<string>\",\n \"dataset_id\": \"<string>\",\n \"config\": {\n \"field_mappings\": {},\n \"normalize_whitespace\": true,\n \"augmentation_config\": {\n \"augmentation_factor\": 1.5,\n \"use_eda\": false,\n \"use_back_translation\": false,\n \"use_paraphrasing\": false,\n \"use_synthesis\": false,\n \"gemini_api_key\": \"<string>\",\n \"synthesis_ratio\": 123,\n \"custom_prompt\": \"<string>\"\n },\n \"split_config\": {\n \"train_split\": \"<string>\",\n \"test_split\": \"<string>\",\n \"type\": \"hf_split\"\n }\n },\n \"dataset_subset\": \"default\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://preprocessing-service-433968519479.us-central1.run.app/datasets/process")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Content-Type"] = 'application/json'
request.body = "{\n \"dataset_name\": \"<string>\",\n \"dataset_id\": \"<string>\",\n \"config\": {\n \"field_mappings\": {},\n \"normalize_whitespace\": true,\n \"augmentation_config\": {\n \"augmentation_factor\": 1.5,\n \"use_eda\": false,\n \"use_back_translation\": false,\n \"use_paraphrasing\": false,\n \"use_synthesis\": false,\n \"gemini_api_key\": \"<string>\",\n \"synthesis_ratio\": 123,\n \"custom_prompt\": \"<string>\"\n },\n \"split_config\": {\n \"train_split\": \"<string>\",\n \"test_split\": \"<string>\",\n \"type\": \"hf_split\"\n }\n },\n \"dataset_subset\": \"default\"\n}"
response = http.request(request)
puts response.read_body{
"dataset_name": "<string>",
"dataset_subset": "<string>",
"dataset_id": "<string>",
"processed_dataset_id": "<string>",
"num_examples": 123,
"created_at": "<string>",
"splits": [
"<string>"
],
"modality": "text",
"full_splits": []
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>"
}
]
}Dataset preprocessing
Process dataset
Process a dataset into ChatML format
The processing converts the dataset to ChatML format using the provided configuration. The configuration can include field mappings that specify either direct column mappings or template strings with column references.
Example field mappings:
{
"system_field": {"type": "template", "value": "You are a helpful assistant."},
"user_field": {"type": "column", "value": "question"},
"assistant_field": {"type": "template", "value": "Answer: {answer}"}
}
POST
/
datasets
/
process
Process Dataset
curl --request POST \
--url https://preprocessing-service-433968519479.us-central1.run.app/datasets/process \
--header 'Content-Type: application/json' \
--data '
{
"dataset_name": "<string>",
"dataset_id": "<string>",
"config": {
"field_mappings": {},
"normalize_whitespace": true,
"augmentation_config": {
"augmentation_factor": 1.5,
"use_eda": false,
"use_back_translation": false,
"use_paraphrasing": false,
"use_synthesis": false,
"gemini_api_key": "<string>",
"synthesis_ratio": 123,
"custom_prompt": "<string>"
},
"split_config": {
"train_split": "<string>",
"test_split": "<string>",
"type": "hf_split"
}
},
"dataset_subset": "default"
}
'import requests
url = "https://preprocessing-service-433968519479.us-central1.run.app/datasets/process"
payload = {
"dataset_name": "<string>",
"dataset_id": "<string>",
"config": {
"field_mappings": {},
"normalize_whitespace": True,
"augmentation_config": {
"augmentation_factor": 1.5,
"use_eda": False,
"use_back_translation": False,
"use_paraphrasing": False,
"use_synthesis": False,
"gemini_api_key": "<string>",
"synthesis_ratio": 123,
"custom_prompt": "<string>"
},
"split_config": {
"train_split": "<string>",
"test_split": "<string>",
"type": "hf_split"
}
},
"dataset_subset": "default"
}
headers = {"Content-Type": "application/json"}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({
dataset_name: '<string>',
dataset_id: '<string>',
config: {
field_mappings: {},
normalize_whitespace: true,
augmentation_config: {
augmentation_factor: 1.5,
use_eda: false,
use_back_translation: false,
use_paraphrasing: false,
use_synthesis: false,
gemini_api_key: '<string>',
synthesis_ratio: 123,
custom_prompt: '<string>'
},
split_config: {train_split: '<string>', test_split: '<string>', type: 'hf_split'}
},
dataset_subset: 'default'
})
};
fetch('https://preprocessing-service-433968519479.us-central1.run.app/datasets/process', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://preprocessing-service-433968519479.us-central1.run.app/datasets/process",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'dataset_name' => '<string>',
'dataset_id' => '<string>',
'config' => [
'field_mappings' => [
],
'normalize_whitespace' => true,
'augmentation_config' => [
'augmentation_factor' => 1.5,
'use_eda' => false,
'use_back_translation' => false,
'use_paraphrasing' => false,
'use_synthesis' => false,
'gemini_api_key' => '<string>',
'synthesis_ratio' => 123,
'custom_prompt' => '<string>'
],
'split_config' => [
'train_split' => '<string>',
'test_split' => '<string>',
'type' => 'hf_split'
]
],
'dataset_subset' => 'default'
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://preprocessing-service-433968519479.us-central1.run.app/datasets/process"
payload := strings.NewReader("{\n \"dataset_name\": \"<string>\",\n \"dataset_id\": \"<string>\",\n \"config\": {\n \"field_mappings\": {},\n \"normalize_whitespace\": true,\n \"augmentation_config\": {\n \"augmentation_factor\": 1.5,\n \"use_eda\": false,\n \"use_back_translation\": false,\n \"use_paraphrasing\": false,\n \"use_synthesis\": false,\n \"gemini_api_key\": \"<string>\",\n \"synthesis_ratio\": 123,\n \"custom_prompt\": \"<string>\"\n },\n \"split_config\": {\n \"train_split\": \"<string>\",\n \"test_split\": \"<string>\",\n \"type\": \"hf_split\"\n }\n },\n \"dataset_subset\": \"default\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://preprocessing-service-433968519479.us-central1.run.app/datasets/process")
.header("Content-Type", "application/json")
.body("{\n \"dataset_name\": \"<string>\",\n \"dataset_id\": \"<string>\",\n \"config\": {\n \"field_mappings\": {},\n \"normalize_whitespace\": true,\n \"augmentation_config\": {\n \"augmentation_factor\": 1.5,\n \"use_eda\": false,\n \"use_back_translation\": false,\n \"use_paraphrasing\": false,\n \"use_synthesis\": false,\n \"gemini_api_key\": \"<string>\",\n \"synthesis_ratio\": 123,\n \"custom_prompt\": \"<string>\"\n },\n \"split_config\": {\n \"train_split\": \"<string>\",\n \"test_split\": \"<string>\",\n \"type\": \"hf_split\"\n }\n },\n \"dataset_subset\": \"default\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://preprocessing-service-433968519479.us-central1.run.app/datasets/process")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Content-Type"] = 'application/json'
request.body = "{\n \"dataset_name\": \"<string>\",\n \"dataset_id\": \"<string>\",\n \"config\": {\n \"field_mappings\": {},\n \"normalize_whitespace\": true,\n \"augmentation_config\": {\n \"augmentation_factor\": 1.5,\n \"use_eda\": false,\n \"use_back_translation\": false,\n \"use_paraphrasing\": false,\n \"use_synthesis\": false,\n \"gemini_api_key\": \"<string>\",\n \"synthesis_ratio\": 123,\n \"custom_prompt\": \"<string>\"\n },\n \"split_config\": {\n \"train_split\": \"<string>\",\n \"test_split\": \"<string>\",\n \"type\": \"hf_split\"\n }\n },\n \"dataset_subset\": \"default\"\n}"
response = http.request(request)
puts response.read_body{
"dataset_name": "<string>",
"dataset_subset": "<string>",
"dataset_id": "<string>",
"processed_dataset_id": "<string>",
"num_examples": 123,
"created_at": "<string>",
"splits": [
"<string>"
],
"modality": "text",
"full_splits": []
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>"
}
]
}Body
application/json
Available options:
upload, huggingface Specifies the preprocessing mode to format the dataset for a specific fine-tuning task.
Available options:
language_modeling, prompt_only, preference Show child attributes
Show child attributes
Response
Successful Response
Available options:
upload, huggingface Available options:
text, vision ⌘I