Intro
I would like to tell you how to scrape Google Search organic results with Node.js.
Preparation
First, we need to create a Node.js project and add npm packages "Axios" and "Cheerio". To do this, in the directory with our project create index.js file, open the command line and enter:
npm init -y
then enter:
npm i axios cheerio
What will be scraped
Process
The following GIF shows the process of selecting Link, Title and Snippet CSS selectors using SelectorGadget Chrome extension.
Code
const cheerio = require("cheerio");
const axios = require("axios");
const searchString = "google";
const encodedString = encodeURI(searchString);
const AXIOS_OPTIONS = {
headers: {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Safari/537.36",
},
};
function getOrganicResults() {
return axios
.get(
`https://www.google.com/search?q=${encodedString}&hl=en&gl=us`,
AXIOS_OPTIONS
)
.then(function ({ data }) {
let $ = cheerio.load(data);
const links = [];
const titles = [];
const snippets = [];
$(".yuRUbf > a").each((i, el) => {
links[i] = $(el).attr("href");
});
$(".yuRUbf > a > h3").each((i, el) => {
titles[i] = $(el).text();
});
$(".IsZvec").each((i, el) => {
snippets[i] = $(el).text().trim();
});
const result = [];
for (let i = 0; i < links.length; i++) {
result[i] = {
link: links[i],
title: titles[i],
snippet: snippets[i],
};
}
console.log(result);
});
}
getOrganicResults();
Output
[
{
link: 'https://www.google.com/',
title: 'Google',
snippet: "Search the world's information, including webpages, images, videos and more. Google has many special features to help you find exactly what you're looking ..."
},
{
link: 'https://blog.google/',
title: 'The Keyword | Google',
snippet: 'Discover all the latest about our products, technology, and Google culture on our official blog.'
},
{
link: 'https://about.google/intl/en_us/products/',
title: "Browse All of Google's Products & Services - Google",
snippet: 'Browse a list of Google products designed to help you work and play, stay organized, get answers, keep in touch, grow your business, and more.'
},
{
link: 'https://about.google/',
title: 'Google - About Google, Our Culture & Company News',
snippet: 'Stay up to date with Google company news and products. Discover stories about our culture, philosophy, and how Google technology is impacting others.'
},
{
link: 'https://m.facebook.com/Google/',
title: 'Google - Home | Facebook',
snippet: 'Google, Mountain View, CA. 28151297 likes · 25276 ... Google, profile picture. Google is on Facebook. To connect with Google, log in or create an account.'
}
]
Using Google Search Organic Results API
SerpApi is a paid API with a free trial of 5,000 searches.
The difference is that all that needs to be done is just to iterate over a ready made, structured JSON instead of coding everything from scratch, and selecting correct selectors which could be time consuming at times.
const SerpApi = require('google-search-results-nodejs');
const search = new SerpApi.GoogleSearch("YOUR_SECRET_KEY"); //To get the key, register on serpapi.com
const params = {
engine: "google",
q: "google",
location: "Austin, Texas, United States",
google_domain: "google.com",
gl: "us",
hl: "en"
};
const callback = function(data) {
console.log(data.organic_results);
};
search.json(params, callback);
Output
organic_results: [
{
position: 1,
title: "Google",
link: "https://www.google.com/",
displayed_link: "https://www.google.com",
snippet:
"Search the world's information, including webpages, images, videos and more. Google has many special features to help you find exactly what you're looking ...",
sitelinks: {
expanded: [
{
title: "Account",
link: "https://www.google.com/account/about/",
snippet:
"You're never more than a tap away from your data and settings. Just ...",
},
{
title: "Google Maps",
link: "https://maps.google.com/",
snippet:
"Get real-time navigation and more in the Maps app. Stay on web ...",
},
{
title: "Images",
link: "https://www.google.com/imghp?hl=en",
snippet: "Google Images. The most comprehensive image search ...",
},
{
title: "My Business",
link: "https://www.google.com/business/",
snippet:
"Your free Business Profile on Google My Business helps you ...",
},
{
title: "Videos",
link: "https://www.google.com/videohp?hl=en",
snippet: "AllImages · Sign in. Videos. REPORT THIS. CANCEL. OK ...",
},
{
title: "Hangouts",
link: "https://hangouts.google.com/",
snippet:
"Use Google Hangouts to keep in touch with one person or a ...",
},
],
},
},
{
position: 2,
title: "The Keyword | Google",
link: "https://blog.google/",
displayed_link: "https://blog.google",
snippet:
"Discover all the latest about our products, technology, and Google culture on our official blog.",
cached_page_link:
"https://webcache.googleusercontent.com/search?q=cache:WqwvZlPx6jkJ:https://blog.google/+&cd=25&hl=en&ct=clnk&gl=us",
related_pages_link:
"https://www.google.com/search?q=related:https://blog.google/+google&sa=X&ved=2ahUKEwj_9IX8vNXxAhXaU80KHS5sCnEQHzAYegQIBBAO",
},
],
Links
Code in the online IDE • SerpApi Playground
Outro
If you want to see how to scrape something using Node.js that I didn't write about yet or you want to see some project made with SerpApi, please write me a message.
Top comments (1)
very cool~!