quickscraper-sdk
v2.0.8
Published
Quick Scraper SDK NodeJS APIs
Downloads
39
Maintainers
Readme
Quick Scraper NodeJS SDK
Register For Free https://www.quickscraper.co
It does supports EcmaScript 8, TypeScript, async-await, Promises !
It does also supports for AWS Lambda like serverless cloud function call.
Please Feel free to create Issue for any help !
Installation
npm install quickscraper-sdk@latest --save
Get Free Account (No Credit Card Required)
- Register yourself here https://app.quickscraper.co/auth/signup
Examples
Typescript
import { QuickScraper } from "quickscraper-sdk";
const QuickScraperClient = new QuickScraper();
try {
const ACCESS_TOKEN = "";
QuickScraperClient.setAccessToken(ACCESS_TOKEN);
const requestUrl = "http://httpbin.org/ip";
const response = await QuickScraperClient.getHtml(requestUrl);
console.log("htmlResponse ", response.data);
console.log("metadata ", response.metadata);
// Manipulate your response here using cheerio
} catch (error) {
console.log("error ", error);
}
Javascript
const QuickScraper = require("quickscraper-sdk");
const QuickScraperClient = new QuickScraper();
try {
const ACCESS_TOKEN = "";
QuickScraperClient.setAccessToken(ACCESS_TOKEN);
const requestUrl = "http://httpbin.org/ip";
const response = await QuickScraperClient.getHtml(requestUrl);
console.log("htmlResponse ", response.data);
console.log("metadata ", response.metadata);
// Manipulate your response here using cheerio
} catch (error) {
console.log("error ", error);
}
Basic Usage
const QuickScraper = require("quickscraper-sdk");
const QuickScraperClient = new QuickScraper();
QuickScraperClient.setAccessToken("YOUR_ACCESS_TOKEN");
const response = await QuickScraperClient.getHtml("http://httpbin.org/ip");
Write a HTML to File
const QuickScraper = require("quickscraper-sdk");
const QuickScraperClient = new QuickScraper();
QuickScraperClient.setAccessToken("YOUR_ACCESS_TOKEN");
await QuickScraperClient.writeHtmlToFile("http://httpbin.org/ip", 'filename.html');
Rendering Javascript
const QuickScraper = require("quickscraper-sdk");
const QuickScraperClient = new QuickScraper();
QuickScraperClient.setAccessToken("YOUR_ACCESS_TOKEN");
const response = await QuickScraperClient.getHtml("http://httpbin.org/ip", {
render: true,
});
Custom Headers
const QuickScraper = require("quickscraper-sdk");
const QuickScraperClient = new QuickScraper();
QuickScraperClient.setAccessToken("YOUR_ACCESS_TOKEN");
const response = await QuickScraperClient.getHtml("https://httpbin.org/headers", {
//This field is added to set cookis,authorizarion token etc from request header.
isKeepHeaders: true,//This value will be true when we want to use header
customRequestHeaders: {
"X-My-Custom-Header": "QS-APP",//Supports values like `'Accept-Encoding': 'gzip,deflate,compress'`
},
});
Geographic Location
const QuickScraper = require("quickscraper-sdk");
const QuickScraperClient = new QuickScraper();
QuickScraperClient.setAccessToken("YOUR_ACCESS_TOKEN");
const response = await QuickScraperClient.getHtml("http://httpbin.org/ip", {
country_code: "US", //Add your Geotargeting country which you are selecting from request page,for example, for Canada value like `CA`,for China value like `CN`
});
console.log(response);
Premium Residential/Mobile Proxy Pools
const QuickScraper = require("quickscraper-sdk");
const QuickScraperClient = new QuickScraper();
QuickScraperClient.setAccessToken("YOUR_ACCESS_TOKEN");
const response = await QuickScraperClient.getHtml("http://httpbin.org/ip", {
premium: true,//Up or premium proxies will be picked
});
console.log(response);
Account Information
const QuickScraper = require("quickscraper-sdk");
const QuickScraperClient = new QuickScraper();
QuickScraperClient.setAccessToken("YOUR_ACCESS_TOKEN");
const usage = await QuickScraperClient.account();
console.log(usage);
Device Type
const QuickScraper = require("quickscraper-sdk");
const QuickScraperClient = new QuickScraper();
QuickScraperClient.setAccessToken("YOUR_ACCESS_TOKEN");
const response = await QuickScraperClient.getHtml("http://httpbin.org/ip", {
device_type: 'mobile' //Add value `mobile` if the device which you are using is mobile and no need to provide this option if want to scrap site as desktop view
});
console.log(response);
Parser Addon
const QuickScraper = require("quickscraper-sdk");
const QuickScraperClient = new QuickScraper();
QuickScraperClient.setAccessToken("YOUR_ACCESS_TOKEN");
const response = await QuickScraperClient.getHtml("http://httpbin.org/ip", {
parserSubscriptionId: 'YOUR_PARSER_SUBSCRIPTION_ID' //Get the parser subscription id from request page when you select parser and paste here, for example value like '4a0360ea-042a-555e-b214-e3054a400f2a'
});
console.log(response);
Webhook Addon
const QuickScraper = require("quickscraper-sdk");
const QuickScraperClient = new QuickScraper();
QuickScraperClient.setAccessToken("YOUR_ACCESS_TOKEN");
const response = await QuickScraperClient.getHtml("http://httpbin.org/ip", {
webhookRequestId: 'WEBHOOK_REQUEST_ID' //Get the webhook id from dashboard and paste here, for example value like `2233-34jkjsd-324jkds-3243kh`
});
console.log(response);
Get JSON from data
const QuickScraper = require("quickscraper-sdk");
const QuickScraperClient = new QuickScraper();
QuickScraperClient.setAccessToken("YOUR_ACCESS_TOKEN");
const response = await QuickScraperClient.getData("http://httpbin.org/ip", {
parserSubscriptionId: 'YOUR_PARSER_SUBSCRIPTION_ID' //Get the parser subscription id from request page when you select parser and paste here, for example value like '4a0360ea-042a-555e-b214-e3054a400f2a'
});
console.log(response);
Submit form data
const QuickScraper = require("quickscraper-sdk");
const QuickScraperClient = new QuickScraper();
QuickScraperClient.setAccessToken("YOUR_ACCESS_TOKEN");
const response = await QuickScraperClient.getData("http://httpbin.org/ip", {
//This field is added when we want to scrap data after login or sign up.
formData: {
formSelector: 'FORM_SELECTOR',//Supports values like `'form[class="unique_class"]'`
submitButtonSelector: 'SUBMIT_BUTTON_SELECTOR',//Supports values like `'button[class="unique_class"]'`
formSelectorScript: 'FORM_SELECTOR_SCRIPT',//Supports values like `document.getElementById("demo")`
submitButtonSelectorScript: 'SUBMIT_BUTTON_SELECTOR_SCRIPT',//Supports values like `document.getElementById("demo")`
formFields: [
{
value: 'VALUE_TO_PASS_IN_FORM',//Add value of form input fields
selectorScript: 'SELECT_INPUT',//Supports values like `'input[class="unique_class"]','span[class="unique_class"]'`
isScript: 'BOOLEAN'//If your selector has JavaScript code then make this true
}
]
},
isKeepFormDataSelection: true,
});
console.log(response);
Add dynamic inputs
const QuickScraper = require("quickscraper-sdk");
const QuickScraperClient = new QuickScraper();
QuickScraperClient.setAccessToken("YOUR_ACCESS_TOKEN");
const response = await QuickScraperClient.getData("http://httpbin.org/ip", {
parserSubscriptionId: 'YOUR_PARSER_SUBSCRIPTION_ID',
//These dynamic inputs field is required when some input fields are required to get data of page
dynamicInputs: [
{
"name":"YOUR_INPUT_NAME",//Name stands for input name field like `Username`,`password`,`limit` etc
"value":"YOUR_VALUE" //Value of name field like `abc`,`123456789`,`25`
}
]
});
console.log(response);
Actions
const QuickScraper = require("quickscraper-sdk");
const QuickScraperClient = new QuickScraper();
QuickScraperClient.setAccessToken("YOUR_ACCESS_TOKEN");
const response = await QuickScraperClient.getHtml("https://www.hilton.com/en/locations/india/?WT.mc_id=zLADA0IN1MB2PSH3GGL4INTCRB5dkt6MULTIBR7en_&epid!_&ebuy!&&&&&gad_source=1&gclid=CjwKCAiAvJarBhA1EiwAGgZl0HWfZEaLc_HWhRFRxTT3gs8pwQvFZKtLPNMj3szkjCbFzlef4ox_XxoCGDYQAvD_BwE&gclsrc=aw.ds", {
customSelectors: [{
"name": "YOUR_VALUE", //Name stands for action name and it supports only 2 actions like `waitForElement` and `click`
'selectorScript': "YOUR_SELECTOR", //Supports values like `'a[href="google.com"]', 'span[class="unique_class"]', document.getElementById("demo")`
'options': null, //This options support timeout feature only like `{"timeout":3000}`
'isScript': false //If your selector has JavaScript code then make this true
}],
keepSelection: true
});
console.log(response);
Scroll To Bottom Of The Page
const QuickScraper = require("quickscraper-sdk");
const QuickScraperClient = new QuickScraper();
QuickScraperClient.setAccessToken("YOUR_ACCESS_TOKEN");
const response = await QuickScraperClient.getHtml("YOUR_REQUEST_URL", {
isScroll: true,
scrollTimeout: 1000 // Consider as milliseconds, Wait for start scraping
});
console.log(response);
Do you need an expert?
Are you finding a developer for your world-class product? If yes, please contact here. Originally by QuickScraper Developers - [email protected].