Puppeteer is a Node library which is providing a high-level API to control Chrome or Chromium over the DevTools Protocol. In simpler terms, being from the same parent gives it greater control and flexibility to work with Chrome, you can do the things manually in the browser using Puppeteer.
Puppeteer’s functionalities easily can challenge to other open-source automation tools.
I. Go to your project directory and execute command: npm i puppeteer
II. Install chokidar package inside your project directory: npm i chokidar
III. Install async package inside your project directory: npm i async
IV. Finally execute npm install command inside project directory
Note: Your system must have latest nodejs installation
const puppeteer = require("puppeteer"); let page = null; let browser = null;
async function login() { // Login function start
let chromeOptions = { headless: false, slowMo: 10, defaultViewport: null, args: ["--no-sandbox", "--disable-setuid-sandbox"] };
browser = await puppeteer.launch(chromeOptions); page = await browser.newPage(); const userAgent = "Mozilla/5.0 (X11; Linux x86_64)" + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36";
await page.setUserAgent(userAgent); await page.goto("http://example.com", { waitUntil: "networkidle0", timeout: 0 }); await page.waitFor(3000);
try {
await page.waitForSelector("#username");
} catch (e) {
console.log(`Error while wait for #username : ${e}`);
}
await page.type("#username", "tester");
await page.type("#password", "tester");
await page.click('button[name="submit"]');
await timeout(4000);
return Promise.resolve(page);
} // Login function end
const timeout = millis => new Promise(resolve => setTimeout(resolve, millis)); module.exports.login = login;
const login = require("./login"); const chokidar = require("chokidar"); const fs = require("fs"); const async = require("async"); const path = __dirname + "/watcherFiles"; const dataPath = __dirname + "/data"; let url = "http://example.com/test"; let statFile = ""; let page = null;
async function startProcess() { page = await login.login(); } const watcher = chokidar.watch("watcherFiles", { persistent: true, ignoreInitial: true, binaryInterval: 10000 });
async function readFile(path) { return new Promise(function(resolve, reject) { fs.readFile(`${path}`, { encoding: "utf-8" }, async function(err, url) { let urlSite = url; if (null === err) { resolve(urlSite); } else { reject(err); } }); }); }
let q = async.queue(function(task, callback) { (async function() { let url = task.name; let result = await main(url); callback(result); })(); }, 1);
function main(url) { return new Promise(async function(resolve, reject) { await page.goto(url, { waitUntil: "networkidle0", timeout: 0 }); if (await page.$("#username")) { await page.type("#username", "tester"); await page.type("#password", "tester"); await page.click('button[name="submit"]'); await timeout(4000); } try { await page.waitForSelector(".some-class-to-load"); } catch (e) { console.log(`Error while wait for .table-responsive : ${e}`); reject(false); } await timeout(4000); resolve(true); }); }
watcher.on("add", async path => { let url = await readFile(path); q.push({name: url}, function(err, data) { if (null === err) { } else { } } ); });
startProcess(); const timeout = millis => new Promise(resolve => setTimeout(resolve, millis));
const fs = require("fs"); const dirPath = __dirname + "/watcherFiles"; async function startProcess() { fs.readdir(dirPath, (err, files) => { if (err) throw err; for (const file of files) { fs.unlink(dirPath + "/" + file, err => { if (err) throw err; }); } }); const file = fs.open( `${dirPath}/ranking_${new Date().getMilliseconds()}.txt`, "w", function(err) { if (err) throw err; console.log("File is created successfully."); } ); } startProcess(); const timeout = millis => new Promise(resolve => setTimeout(resolve, millis));
By using puppeteers, async and watcher we can automate any website and we can manage sessions as well.