iver-http

http数据抓取模块

Install

npm install --save iver-http

Method

• PromiseRequest - Request Promise构造
• httpGet - 重复发送
• getListArr - 列表抓取
• getDetailArr - 详情页抓取
• PromiseHttpGetByUrls - 自定义抓取

方法的参数，可以参考下面的使用。

Usage

const fs = require("fs");
const { getListArr, getDetailArr } = require("iver-http")
async function getHtml() {
  const urls = [..."2".repeat(2)].map((_, i) => `https://wh.lianjia.com/ershoufang/pg${i + 1}/`)
  const list = await getListArr({
    urls,
    $listStr: `$(".sellListContent li")`,
    $mapObj: {
      "title": `$(ele).find("a").eq(1).html()`,
      "area": `$(ele).find("a").eq(2).html()`,
      "place": `$(ele).find("a").eq(3).html()`,
      "url": `$(ele).find("a").eq(1).attr("href")`,
    },
    logfunc(i, resObj, len) {
      console.log(`-共${len}条--${i}--${resObj.desc}-${resObj.url}-`);
    }
  })

  // fs.writeFileSync("./list.json", JSON.stringify(list, null, 2))
  console.log("list success", list.length)
  const details = await getDetailArr({
    urls: list.map(item => item.url),
    $mapObj: {
      title: `$(".sellDetailHeader .main").html()`,
      price: `$(".overview .content .price .total").text()`,
      roomMain: `$(".overview .content .houseInfo .room .mainInfo").html() `,
      roomSub: `$(".overview .content .houseInfo .room .subInfo").html()`,
      typeMain: `$(".overview .content .houseInfo .type .mainInfo").html()`,
      typeSub: `$(".overview .content .houseInfo .type .subInfo").html()`,
      areaMain: `$(".overview .content .houseInfo .area .mainInfo").html()`,
      areaSub: `$(".overview .content .houseInfo .area .subInfo").html()`,
    },
    logfunc(i, resObj, len) {
      console.log(`-共${len}条--${i}--${resObj.desc}-${resObj.url}-`);
    }
  })
  // fs.writeFileSync("./details.json", JSON.stringify(details, null, 2))
  console.log("details success", details.length)
}
getHtml();

More

....