README
Scraping-bot.io
Getting started
API endpoint : http://api.scraping-bot.io/scrape use POST method !
Hey ! Just getting started with Scraping Bot API ? No problem, we will introduce you to this magical world here you can find scraping code samples!
In case a request fails we will return a 200 code with a JSON populated with an error, you may retry the request Make sure to catch these errors! They will occur for hard to scrape websites.
If you exceed your plan concurrent connection limit, the API will respond with a 402 status code, this can be solved by slowing down your request rate
If you exceed your plan api calls, the API will respond with a 402 status code, this can be solved by upgrading your plan.
There is no overage allowed on the free plan, if you exceed 100 requests per month on the free plan, you will receive a 402 error.
Note that requests on Google count for 10 request credits and for Amazon they will count for 5 resquest credits.
Each request will return a string with raw JSON
the error field is null by default and is filled with a string if some error occurred
the data field contains by default all fields listed below
siteHtml is always null you can get it with advanced options example here
{
"error": null,
"data": {
"title": "Apple iPhone XR 64GB Red Unlocked A2105 GSM SEALED BOX- 1 Year Apple Warranty |",
"description": "Apple iPhone XR. 1 YEAR APPLE CARE WARRANTY.",
"image": "https://www.scraping-bot.io/iphone_example_ebay_files/s-l500.png",
"price": 689,
"shippingFees": 18,
"currency": "GBP",
"isInStock": true,
"category": null,
"siteURL": "https://www.ebay.co.uk/itm/Apple-iPhone-XR-64GB-Red-Unlocked-A2105-GSM-SEALED-BOX-1-Year-Apple-Warranty-/123902112947",
"siteHtml": null,
"statusCode": null,
"EAN13": null,
"ASIN": null,
"ISBN": null,
"color": null,
"brand": null,
"host": "www.ebay.co.uk"
}
}
Basic usage
You are using Scraping Bot for a basic usage ? You’ll find more information just here :
NodeJs
#!/bin/bash
url='https://www.scraping-bot.io/example-ebay.html'
username='yourUsername'
api_key='yourApiKey'
auth=$(echo -ne "$username:$api_key" | base64);
curl -X POST \
http://api.scraping-bot.io/scrape \
-H "Authorization: Basic $auth" \
-H "Content-Type: application/json" \
-d "{\"url\":\"$url\"}"
Result :
{
"error": null,
"data": {
"title": "Apple iPhone XR 64GB Red Unlocked A2105 GSM SEALED BOX- 1 Year Apple Warranty |",
"description": "Apple iPhone XR. 1 YEAR APPLE CARE WARRANTY.",
"image": "https://www.scraping-bot.io/iphone_example_ebay_files/s-l500.png",
"price": 689,
"shippingFees": 18,
"currency": "GBP",
"isInStock": true,
"category": null,
"siteURL": "https://www.ebay.co.uk/itm/Apple-iPhone-XR-64GB-Red-Unlocked-A2105-GSM-SEALED-BOX-1-Year-Apple-Warranty-/123902112947",
"siteHtml": null,
"statusCode": null,
"EAN13": null,
"ASIN": null,
"ISBN": null,
"color": null,
"brand": null,
"host": "www.ebay.co.uk"
}
}
Bash
#!/bin/bash
url='https://www.scraping-bot.io/example-ebay.html'
username='yourUsername'
api_key='yourApiKey'
auth=$(echo -ne "$username:$api_key" | base64);
curl -X POST \
http://api.scraping-bot.io/scrape \
-H "Authorization: Basic $auth" \
-H "Content-Type: application/json" \
-d "{\"url\":\"$url\"}"
Result :
{
"error": null,
"data": {
"title": "Apple iPhone XR 64GB Red Unlocked A2105 GSM SEALED BOX- 1 Year Apple Warranty |",
"description": "Apple iPhone XR. 1 YEAR APPLE CARE WARRANTY.",
"image": "https://www.scraping-bot.io/iphone_example_ebay_files/s-l500.png",
"price": 689,
"shippingFees": 18,
"currency": "GBP",
"isInStock": true,
"category": null,
"siteURL": "https://www.ebay.co.uk/itm/Apple-iPhone-XR-64GB-Red-Unlocked-A2105-GSM-SEALED-BOX-1-Year-Apple-Warranty-/123902112947",
"siteHtml": null,
"statusCode": null,
"EAN13": null,
"ASIN": null,
"ISBN": null,
"color": null,
"brand": null,
"host": "www.ebay.co.uk"
}
}
Php
"http://api.scraping-bot.io/scrape",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => "{\"url\":\"https://www.scraping-bot.io/example-ebay.html\"}",
CURLOPT_HTTPHEADER => array(
"Authorization: Basic ".$auth,
"Content-Type: application/json"
),
));
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
$info = curl_getinfo($curl);
if($info["http_code"]>399){
echo "HTTP Error #:" . $response;
}else{
echo $response;
}
}
Result :
{
"error": null,
"data": {
"title": "Apple iPhone XR 64GB Red Unlocked A2105 GSM SEALED BOX- 1 Year Apple Warranty |",
"description": "Apple iPhone XR. 1 YEAR APPLE CARE WARRANTY.",
"image": "https://www.scraping-bot.io/iphone_example_ebay_files/s-l500.png",
"price": 689,
"shippingFees": 18,
"currency": "GBP",
"isInStock": true,
"category": null,
"siteURL": "https://www.ebay.co.uk/itm/Apple-iPhone-XR-64GB-Red-Unlocked-A2105-GSM-SEALED-BOX-1-Year-Apple-Warranty-/123902112947",
"siteHtml": null,
"statusCode": null,
"EAN13": null,
"ASIN": null,
"ISBN": null,
"color": null,
"brand": null,
"host": "www.ebay.co.uk"
}
}
Python
import requests
url='https://www.scraping-bot.io/example-ebay.html'
username = 'yourUsername'
apiKey = 'yourApiKey'
authString = username+":"+apiKey;
encodedBytes = base64.b64encode(authString.encode("utf-8"))
encodedAuth = str(encodedBytes, "utf-8")
apiUrl = "http://api.scraping-bot.io/scrape"
payload = "{\n \"url\":\""+url+"\"\n}"
headers = {
'Content-Type': "application/json",
'Authorization': "Basic "+encodedAuth
}
response = requests.request("POST", apiUrl, data=payload, headers=headers)
print(response.text)
Result :
{
"error": null,
"data": {
"title": "Apple iPhone XR 64GB Red Unlocked A2105 GSM SEALED BOX- 1 Year Apple Warranty |",
"description": "Apple iPhone XR. 1 YEAR APPLE CARE WARRANTY.",
"image": "https://www.scraping-bot.io/iphone_example_ebay_files/s-l500.png",
"price": 689,
"shippingFees": 18,
"currency": "GBP",
"isInStock": true,
"category": null,
"siteURL": "https://www.ebay.co.uk/itm/Apple-iPhone-XR-64GB-Red-Unlocked-A2105-GSM-SEALED-BOX-1-Year-Apple-Warranty-/123902112947",
"siteHtml": null,
"statusCode": null,
"EAN13": null,
"ASIN": null,
"ISBN": null,
"color": null,
"brand": null,
"host": "www.ebay.co.uk"
}
}
Ruby
require 'uri'
require 'net/http'
url='https://www.scraping-bot.io/example-ebay.html'
username='yourUsername'
api_key='yourApiKey'
apiUrl = URI("http://api.scraping-bot.io/scrape")
http = Net::HTTP.new(apiUrl.host, apiUrl.port)
request = Net::HTTP::Post.new(apiUrl)
request["Content-Type"] = 'application/json'
request["Authorization"] = 'Basic eW91clVzZXJuYW1lOnlvdXJBcGlLZXk='
request["cache-control"] = 'no-cache'
request["Postman-Token"] = '24567912-050b-4082-833a-9115b2376f6f'
request.body = "{\n \"url\":\"https://www.scraping-bot.io/example-ebay.html\"\n}"
response = http.request(request)
puts response.read_body
Result :
{
"error": null,
"data": {
"title": "Apple iPhone XR 64GB Red Unlocked A2105 GSM SEALED BOX- 1 Year Apple Warranty |",
"description": "Apple iPhone XR. 1 YEAR APPLE CARE WARRANTY.",
"image": "https://www.scraping-bot.io/iphone_example_ebay_files/s-l500.png",
"price": 689,
"shippingFees": 18,
"currency": "GBP",
"isInStock": true,
"category": null,
"siteURL": "https://www.ebay.co.uk/itm/Apple-iPhone-XR-64GB-Red-Unlocked-A2105-GSM-SEALED-BOX-1-Year-Apple-Warranty-/123902112947",
"siteHtml": null,
"statusCode": null,
"EAN13": null,
"ASIN": null,
"ISBN": null,
"color": null,
"brand": null,
"host": "www.ebay.co.uk"
}
}
Advanced options
You choose the best usage of Scraping Bot ? I knew that you were the best. Here you can find details about the options you can choose.
Options :
getHtml : Boolean, If you want to get full html body you need to set this option to true defaults to false
useChrome : Boolean, set this option to true if you want to use headless chrome that is able to render Javascript and get full result WARNING this option consumes two api calls
NodeJs
var request = require('request');
var username = "yourUsername",
apiKey = "yourApiKey",
url = "https://www.scraping-bot.io/example-ebay.html",
auth = "Basic " + new Buffer(username + ":" + apiKey).toString("base64");
request(
{
method: 'POST',
url: 'http://api.scraping-bot.io/scrape',
json: {
url: url,
options: {
getHtml:true, //if you want to get the full html of the page
useChrome:true, //if you want to use headless chrome WARNING two api calls wiil be consumed for this option
}
},
headers: {
Accept: 'application/json',
Authorization : auth
},
},
function(error, response, body) {
console.log(body);
}
);
Bash
#!/bin/bash
url='https://www.scraping-bot.io/example-ebay.html'
username='yourUsername'
api_key='yourApiKey'
auth=$(echo -ne "$username:$api_key" | base64);
curl -X POST \
http://api.scraping-bot.io/scrape \
-H "Authorization: Basic $auth" \
-H "Content-Type: application/json" \
-d "{\"url\":\"$url\",\"options\":{\"getHtml\":\"true\",\"useChrome\":\"true\"}}"
Php
"https://www.scraping-bot.io/example-ebay.html",
'options' => array(
"getHtml" => true,
"useChrome" => true
)
);
$json = json_encode($postParams);
$curl = curl_init();
curl_setopt_array($curl, array(
CURLOPT_URL => "http://api.scraping-bot.io/scrape",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => $json,
CURLOPT_HTTPHEADER => array(
"Authorization: Basic ".$auth,
"Content-Type: application/json"
),
));
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}
Followers: 2
Resources:
Product Website API Creator:
Perrine
Perrine
Log In to Rate API
Rating: 5 - Votes: 1