udger-nodejs

v1.5.0

Published

2 years ago

NodeJS User-Agent String Parser based on Udger SQLite databases https://udger.com/products/local_parser

Downloads

167

0High
0Medium
0Low

eviltik

udger user-agent device-detector bot-detection mobile-detection bot bots antibot antibots ipv4 ipv6 datacenter crawler

udger-nodejs

Udger client for NodeJS (data ver. 3)

Local parser is very fast and accurate useragent string detection solution. Enables developers to locally install and integrate a highly-scalable product. We provide the detection of the devices (personal computer, tablet, Smart TV, Game console etc.), operating system, client SW type (browser, e-mail client etc.) and devices market name (example: Sony Xperia Tablet S, Nokia Lumia 820 etc.). It also provides information about IP addresses (Public proxies, VPN services, Tor exit nodes, Fake crawlers, Web scrapers, Datacenter name .. etc.)

Tested with more the 50.000 unique user agents.
Up to date data provided by https://udger.com/

Requirements

nodejs >= 8.9.0
datafile v3 (udgerdb_v3.dat) from https://data.udger.com/

Features

Fast
LRU cache
Released under the MIT

Install

npm install udger-nodejs

Usage

You should review the included examples in examples/ and test/ directory.

Here's a quick example:

const udgerParser = require('udger-nodejs')('./udgerdb_v3.dat');

udgerParser.set({
    ua:'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36',
    ip:'2A02:598:7000:116:0:0:0:101'
});

let ret = udgerParser.parse();

// beautify json output with 4 spaces indent
console.log(JSON.stringify(ret, null, 4));

Result

{
    "user_agent": {
        "ua_string": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36",
        "ua_class": "Browser",
        "ua_class_code": "browser",
        "ua": "Chrome 62.0.3202.94",
        "ua_version": "62.0.3202.94",
        "ua_version_major": "62",
        "ua_uptodate_current_version": "62",
        "ua_family": "Chrome",
        "ua_family_code": "chrome",
        "ua_family_homepage": "http://www.google.com/chrome/",
        "ua_family_vendor": "Google Inc.",
        "ua_family_vendor_code": "google_inc",
        "ua_family_vendor_homepage": "https://www.google.com/about/company/",
        "ua_family_icon": "chrome.png",
        "ua_family_icon_big": "chrome_big.png",
        "ua_family_info_url": "https://udger.com/resources/ua-list/browser-detail?browser=Chrome",
        "ua_engine": "WebKit/Blink",
        "os": "Windows 10",
        "os_code": "windows_10",
        "os_homepage": "https://en.wikipedia.org/wiki/Windows_10",
        "os_icon": "windows10.png",
        "os_icon_big": "windows10_big.png",
        "os_info_url": "https://udger.com/resources/ua-list/os-detail?os=Windows 10",
        "os_family": "Windows",
        "os_family_code": "windows",
        "os_family_vendor": "Microsoft Corporation.",
        "os_family_vendor_code": "microsoft_corporation",
        "os_family_vendor_homepage": "https://www.microsoft.com/about/",
        "device_class": "Desktop",
        "device_class_code": "desktop",
        "device_class_icon": "desktop.png",
        "device_class_icon_big": "desktop_big.png",
        "device_class_info_url": "https://udger.com/resources/ua-list/device-detail?device=Desktop",
        "device_marketname": "",
        "device_brand": "",
        "device_brand_code": "",
        "device_brand_homepage": "",
        "device_brand_icon": "",
        "device_brand_icon_big": "",
        "device_brand_info_url": "",
        "crawler_last_seen": "",
        "crawler_category": "",
        "crawler_category_code": "",
        "crawler_respect_robotstxt": ""
    },
    "ip_address": {
        "ip": "2a02:598:7000:116:0:0:0:101",
        "ip_ver": 6,
        "ip_classification": "Crawler",
        "ip_classification_code": "crawler",
        "ip_hostname": "",
        "ip_last_seen": "2016-02-12 04:28:56",
        "ip_country": "Czech Republic",
        "ip_country_code": "CZ",
        "ip_city": "Prague",
        "crawler_name": "SeznamBot/3.2-test1",
        "crawler_ver": "3.2-test1",
        "crawler_ver_major": "3",
        "crawler_family": "SeznamBot",
        "crawler_family_code": "seznambot",
        "crawler_family_homepage": "http://napoveda.seznam.cz/en/seznambot-intro/",
        "crawler_family_vendor": "Seznam.cz, a.s.",
        "crawler_family_vendor_code": "seznam-cz_as",
        "crawler_family_vendor_homepage": "http://onas.seznam.cz/",
        "crawler_family_icon": "seznam.png",
        "crawler_family_info_url": "https://udger.com/resources/ua-list/bot-detail?bot=SeznamBot#id12590",
        "crawler_last_seen": "2016-08-31 15:19:38",
        "crawler_category": "Search engine bot",
        "crawler_category_code": "search_engine_bot",
        "crawler_respect_robotstxt": "yes",
        "datacenter_name": "Seznam.cz",
        "datacenter_name_code": "seznam_cz",
        "datacenter_homepage": "http://onas.seznam.cz/"
    },
    "from_cache": false
}

Result formats

By default, JSON Udger format is used. The JSON Udger format is coming from the PHP Parser (compatibility) Others JSON format are available, by passing options in udger.parse() function.

Native Udger JSON Format (default)

Usage

udgerParser.parse();

Result

{
    "ip_address": {
        "ip": "66.249.64.73",
        "ip_ver": 4,
        "ip_classification": "Crawler",
        "ip_classification_code": "crawler",
        "ip_hostname": "crawl-66-249-64-73.googlebot.com",
        "ip_last_seen": "2016-10-02 09:16:57",
        "ip_country": "United States",
        "ip_country_code": "US",
        "ip_city": "Mountain View",
        "crawler_name": "Googlebot/2.1",
        "crawler_ver": "2.1",
        "crawler_ver_major": "2",
        "crawler_family": "Googlebot",
        "crawler_family_code": "googlebot",
        "crawler_family_homepage": "http://www.google.com/bot.html",
        "crawler_family_vendor": "Google Inc.",
        "crawler_family_vendor_code": "google_inc",
        "crawler_family_vendor_homepage": "https://www.google.com/about/company/",
        "crawler_family_icon": "bot_googlebot.png",
        "crawler_family_info_url": "https://udger.com/resources/ua-list/bot-detail?bot=Googlebot#id31",
        "crawler_last_seen": "2017-01-06 17:52:46",
        "crawler_category": "Search engine bot",
        "crawler_category_code": "search_engine_bot",
        "crawler_respect_robotstxt": "yes",
        "datacenter_name": "Google sites",
        "datacenter_name_code": "googgle_sites",
        "datacenter_homepage": "http://sites.google.com/"
    }
}

Compact JSON Format

Usage

udgerParser.parse({json:true});

Result

{
    "ipAddress": {
        "ip": "66.249.64.73",
        "classification": "crawler",
        "lastSeen": "2016-10-02 09:16:57",
        "hostname": "crawl-66-249-64-73.googlebot.com",
        "geo": {
            "country": {
                "name": "United States",
                "code": "US"
            },
            "city": "Mountain View"
        },
        "crawler": {
            "name": "Googlebot/2.1",
            "family": "googlebot",
            "category": "search_engine_bot",
            "lastSeen": "2017-01-06 17:52:46"
        },
        "datacenter": "googgle_sites"
    }
}

Full JSON Format

Usage

udgerParser.parse({json:true, full:true});

Result

{
    "ipAddress": {
        "ip": "66.249.64.73",
        "version": 4,
        "classification": {
            "name": "Crawler",
            "code": "crawler"
        },
        "lastSeen": "2016-10-02 09:16:57",
        "hostname": "crawl-66-249-64-73.googlebot.com",
        "geo": {
            "country": {
                "name": "United States",
                "code": "US"
            },
            "city": "Mountain View"
        },
        "crawler": {
            "name": "Googlebot/2.1",
            "version": {
                "current": "2.1",
                "major": "2"
            },
            "family": {
                "name": "Googlebot",
                "code": "googlebot",
                "homepage": "http://www.google.com/bot.html",
                "vendor": {
                    "name": "Google Inc.",
                    "code": "google_inc",
                    "homepage": "https://www.google.com/about/company/"
                },
                "icon": "bot_googlebot.png",
                "infoUrl": "https://udger.com/resources/ua-list/bot-detail?bot=Googlebot#id31"
            },
            "lastSeen": "2017-01-06 17:52:46",
            "category": {
                "name": "Search engine bot",
                "code": "search_engine_bot"
            },
            "respectRobotsTxt": "search_engine_bot"
        },
        "datacenter": {
            "name": "Google sites",
            "code": "googgle_sites",
            "homepage": "http://sites.google.com/"
        }
    },
    "fromCache": false
}

LRU Cache

By default, cache is disable. To enable cache, just add this line BEFORE using udgerParser.set():

// by default, cache size is 4000 keys (a key can be an UA, or UA+IP)
// you can modify this limit
udgerParser.setCacheSize(1000);
udgerParser.setCacheEnable(true)

When a record is coming from the cache, the "from_cache" attribute in the response is "true"

Helpers

randomUACrawlers

// return callback with random ua_string from udger_crawler_list table
//  [
//      { ua_string: 'Googlebot/2.1 (+http://www.google.com/bot.html)' },
//      { ua_string: 'Mozilla/5.0 (compatible; SeznamBot/3.2-test1; +http://napoveda.seznam.cz/en/seznambot-intro/)' },
//      ......
//  ]

udgerParser.randomUACrawlers(10, (err, results) => {});

randomUAClientsRegex

// return callback with random regstring from udger_client_regex table
//  [
//      { regstring: '/^Microsoft Office\\/16.*Microsoft Outlook Mail 16/si' },
//      { regstring: '/Mozilla.*Linux.*Android.*WebKit.*Version\\/([0-9\\.]+)/si' },
//      ......
//  ]

udgerParser.randomUAClientsRegex(10, (err, results) => {});

randomUAClients

// return callback with random generated UA strings based in regstring (udger_client_regex)
//  [
//      {
//          regstring: '/^Mozilla.*MSIE.*Windows Phone.*IEMobile\\/([0-9\\.]+)/si',
//          randomUA: 'MozillaMSIEQSs-0Windows PhoneTD7Y9IEMobile/24304'
//      },
//      {
//          regstring: '/mozilla.*rv:[0-9\\.]+.*servo.*firefox\\/([0-9a-z\\+\\-\\.]+).*/si',
//          randomUA: 'mozillatWrv:90221.GeservoTKqzfirefox/y-l+h'
//      }
//      ......
//  ]

udgerParser.randomUAClients(10, (err, results) => {});

randomIPv4

// return callback with random ipv4 matching XXX.XXX.XXX.XXX from udger_ip_list (identified as "bad" IPs)
//  [
//      { ip: '108.61.199.93' },
//      { ip: '66.249.64.73' }
//      .....
//  ]

udgerParser.randomIPv4(10, (err, results) => {});

getUAClientsClassification

// return callback with data from udger_client_class table
//  [
//      { client_classification: 'Browser', client_classification_code: 'browser' },
//      { client_classification: 'Offline browser', client_classification_code: 'offline_browser' },
//      .....
// ]

udgerParser.getUAClientsClassification((err, results) => {});

getUACrawlersClassification

// return callback with data from udger_crawler_class table
//  [
//      { crawler_classification: 'Uncategorised', crawler_classification_code: 'uncategorised' },
//      { crawler_classification: 'Search engine bot', crawler_classification_code: 'search_engine_bot' },
//      .....
//  ]

udgerParser.getUACrawlersClassification((err, results) => {});

getUACrawlersFamilies

// return callback with data from mixed udger_crawler_class, udger_crawler_list tables
//  [
//      {
//          family_code: 'googlebot',
//          crawler_classification_code: 'search_engine_bot'
//      },
//      ....
//  ]

udgerParser.getUACrawlersFamilies((err, results) => {});

getIpsClassification

// return callback with data from udger_ip_class table
//  [
//      { ip_classification: 'Tor exit node', ip_classification_code: 'tor_exit_node' },
//      { ip_classification: 'Fake crawler', ip_classification_code: 'fake_crawler' },
//      .....
//  ]

udgerParser.getIpsClassification((err, results) => {});

getDatabaseInfo

// return callback with single record from udger_db_info table
//  {
//      version: '20170106-01',
//      information: 'Data v3 for Local parser - test data, no full database',
//      lastupdate: 1483690193
//  }

udgerParser.getDatabaseInfo((err, result) => {});

Running tests

npm test

Automatic updates download

for autoupdate data use Udger data updater (https://udger.com/support/documentation/?doc=62)

Documentation for programmers

https://udger.com/pub/documentation/parser/NodeJS/html/

Author

The Udger.com Team ([email protected])

old v2 format

This module does not support v2 format

Published

Vulnerabilities

Links

Maintainers

Keywords

Readme

udger-nodejs

Udger client for NodeJS (data ver. 3)

Requirements

Features

Install

Usage

Result formats

Native Udger JSON Format (default)

Usage

Result

Compact JSON Format

Usage

Result

Full JSON Format

Usage

Result

LRU Cache

Helpers

randomUACrawlers

randomUAClientsRegex

randomUAClients

randomIPv4

getUAClientsClassification

getUACrawlersClassification

getUACrawlersFamilies

getIpsClassification

getDatabaseInfo

Running tests

Automatic updates download

Documentation for programmers

Author

old v2 format