aheh/bit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(cli): --update-data download UA Parser and GeoLite2

Browse Source
This commit is contained in:
sjdonado
2025-03-16 13:40:33 +01:00
parent b538a379d1
commit cd8e2433a5
4 changed files with 239 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/controllers/link.cr
+1 -1
View File
@@ -4,7 +4,7 @@ require "user_agent_parser"
require "../lib/controller.cr"
require "../lib/ip_lookup"
UserAgent.load_regexes(File.read("data/regexes.yaml"))
UserAgent.load_regexes(File.read("data/uap_core_regexes.yaml"))
IpLookup.load_mmdb("data/GeoLite2-Country.mmdb")
module App::Controllers::Link
app/services/cli.cr
+85
View File
@@ -1,3 +1,6 @@
require "file_utils"
require "http/client"
require "../config/*"
require "../lib/*"
require "../models/*"
@@ -53,4 +56,86 @@ module App::Services::Cli
puts "Admin setup skipped: Missing ADMIN_NAME or ADMIN_API_KEY environment variables."
end
end
def self.update_uap_regexes
puts "Downloading User-Agent Parser core regexes..."
FileUtils.mkdir_p("data")
url = "https://raw.githubusercontent.com/ua-parser/uap-core/master/regexes.yaml"
output_file = "data/uap_core_regexes.yaml"
begin
http_get_with_redirect(url) do |response|
File.write(output_file, response.body_io.gets_to_end)
end
puts "User-Agent regexes downloaded to #{output_file}"
rescue e
puts "Error: Failed to download UAP core regexes: #{e.message}"
end
end
def self.download_geolite_db
puts "Downloading GeoLite2 Country database..."
FileUtils.mkdir_p("data")
url = "https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-Country.mmdb"
output_file = "data/GeoLite2-Country.mmdb"
begin
File.open(output_file, "wb") do |file|
http_get_with_redirect(url) do |response|
IO.copy(response.body_io, file)
end
end
puts "GeoLite2 database downloaded to #{output_file}"
rescue e
puts "Error: Failed to download GeoLite2 database: #{e.message}"
end
end
private def self.http_get_with_redirect(url : String, max_redirects = 5)
redirects = 0
while redirects < max_redirects
uri = URI.parse(url)
client = HTTP::Client.new(uri)
success = false
follow_redirect = false
redirect_url = nil
begin
client.get(uri.request_target) do |response|
case response.status_code
when 200
yield response
success = true
when 301, 302
if new_location = response.headers["Location"]?
puts "Following redirect to: #{new_location}"
redirect_url = new_location
follow_redirect = true
else
raise "Received redirect status but no Location header"
end
else
raise "Failed request with status code: #{response.status_code}"
end
end
ensure
client.close
end
return if success
if follow_redirect && redirect_url
url = redirect_url
redirects += 1
else
break
end
end
raise "Too many redirects (#{max_redirects})"
end
end
data/regexes.yaml → data/uap_core_regexes.yaml
+141 -23
View File
@@ -148,11 +148,11 @@ user_agent_parsers:
family_replacement: 'Pinterestbot'
# Bots
- regex: '(CSimpleSpider|Cityreview Robot|CrawlDaddy|CrawlFire|Finderbots|Index crawler|Job Roboter|KiwiStatus Spider|Lijit Crawler|QuerySeekerSpider|ScollSpider|Trends Crawler|USyd-NLP-Spider|SiteCat Webbot|BotName\/\$BotVersion|123metaspider-Bot|1470\.net crawler|50\.nu|8bo Crawler Bot|Aboundex|Accoona-[A-z]{1,30}-Agent|AdsBot-Google(?:-[a-z]{1,30}|)|altavista|AppEngine-Google|archive.{0,30}\.org_bot|archiver|Ask Jeeves|[Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]{1,30})(?:-[A-Za-z]{1,30}|)|bingbot|BingPreview|blitzbot|BlogBridge|Bloglovin|BoardReader Blog Indexer|BoardReader Favicon Fetcher|boitho.com-dc|BotSeer|BUbiNG|\b\w{0,30}favicon\w{0,30}\b|\bYeti(?:-[a-z]{1,30}|)|Catchpoint(?: bot|)|[Cc]harlotte|Checklinks|clumboot|Comodo HTTP\(S\) Crawler|Comodo-Webinspector-Crawler|ConveraCrawler|CRAWL-E|CrawlConvera|Daumoa(?:-feedfetcher|)|Feed Seeker Bot|Feedbin|findlinks|Flamingo_SearchEngine|FollowSite Bot|furlbot|Genieo|gigabot|GomezAgent|gonzo1|(?:[a-zA-Z]{1,30}-|)Googlebot(?:-[a-zA-Z]{1,30}|)|Google SketchUp|grub-client|gsa-crawler|heritrix|HiddenMarket|holmes|HooWWWer|htdig|ia_archiver|ICC-Crawler|Icarus6j|ichiro(?:/mobile|)|IconSurf|IlTrovatore(?:-Setaccio|)|InfuzApp|Innovazion Crawler|InternetArchive|IP2[a-z]{1,30}Bot|jbot\b|KaloogaBot|Kraken|Kurzor|larbin|LEIA|LesnikBot|Linguee Bot|LinkAider|LinkedInBot|Lite Bot|Llaut|lycos|Mail\.RU_Bot|masscan|masidani_bot|Mediapartners-Google|Microsoft .{0,30} Bot|mogimogi|mozDex|MJ12bot|msnbot(?:-media {0,2}|)|msrbot|Mtps Feed Aggregation System|netresearch|Netvibes|NewsGator[^/]{0,30}|^NING|Nutch[^/]{0,30}|Nymesis|ObjectsSearch|OgScrper|Orbiter|OOZBOT|PagePeeker|PagesInventory|PaxleFramework|Peeplo Screenshot Bot|PHPCrawl|PlantyNet_WebRobot|Pompos|Qwantify|Read%20Later|Reaper|RedCarpet|Retreiver|Riddler|Rival IQ|scooter|Scrapy|Scrubby|searchsight|seekbot|semanticdiscovery|SemrushBot|Simpy|SimplePie|SEOstats|SimpleRSS|SiteCon|Slackbot-LinkExpanding|Slack-ImgProxy|Slurp|snappy|Speedy Spider|Squrl Java|Stringer|TheUsefulbot|ThumbShotsBot|Thumbshots\.ru|Tiny Tiny RSS|Twitterbot|WhatsApp|URL2PNG|Vagabondo|VoilaBot|^vortex|Votay bot|^voyager|WASALive.Bot|Web-sniffer|WebThumb|WeSEE:[A-z]{1,30}|WhatWeb|WIRE|WordPress|Wotbox|www\.almaden\.ibm\.com|Xenu(?:.s|) Link Sleuth|Xerka [A-z]{1,30}Bot|yacy(?:bot|)|YahooSeeker|Yahoo! Slurp|Yandex\w{1,30}|YodaoBot(?:-[A-z]{1,30}|)|YottaaMonitor|Yowedo|^Zao|^Zao-Crawler|ZeBot_www\.ze\.bz|ZooShot|ZyBorg|ArcGIS Hub Indexer)(?:[ /]v?(\d+)(?:\.(\d+)(?:\.(\d+)|)|)|)'
- regex: '(CSimpleSpider|Cityreview Robot|CrawlDaddy|CrawlFire|Finderbots|Index crawler|Job Roboter|KiwiStatus Spider|Lijit Crawler|QuerySeekerSpider|ScollSpider|Trends Crawler|USyd-NLP-Spider|SiteCat Webbot|BotName\/\$BotVersion|123metaspider-Bot|1470\.net crawler|50\.nu|8bo Crawler Bot|Aboundex|Accoona-[A-z]{1,30}-Agent|AdsBot-Google(?:-[a-z]{1,30}|)|altavista|AppEngine-Google|archive.{0,30}\.org_bot|archiver|Ask Jeeves|[Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]{1,30})(?:-[A-Za-z]{1,30}|)|bingbot|BingPreview|blitzbot|BlogBridge|Bloglovin|BoardReader Blog Indexer|BoardReader Favicon Fetcher|boitho.com-dc|BotSeer|BUbiNG|\b\w{0,30}favicon\w{0,30}\b|\bYeti(?:-[a-z]{1,30}|)|Catchpoint(?: bot|)|[Cc]harlotte|Checklinks|clumboot|Comodo HTTP\(S\) Crawler|Comodo-Webinspector-Crawler|ConveraCrawler|CRAWL-E|CrawlConvera|Daumoa(?:-feedfetcher|)|Feed Seeker Bot|Feedbin|findlinks|Flamingo_SearchEngine|FollowSite Bot|furlbot|Genieo|gigabot|GomezAgent|gonzo1|(?:[a-zA-Z]{1,30}-|)Googlebot(?:-[a-zA-Z]{1,30}|)|GoogleOther|Google SketchUp|grub-client|gsa-crawler|heritrix|HiddenMarket|holmes|HooWWWer|htdig|ia_archiver|ICC-Crawler|Icarus6j|ichiro(?:/mobile|)|IconSurf|IlTrovatore(?:-Setaccio|)|InfuzApp|Innovazion Crawler|InternetArchive|IP2[a-z]{1,30}Bot|jbot\b|KaloogaBot|Kraken|Kurzor|larbin|LEIA|LesnikBot|Linguee Bot|LinkAider|LinkedInBot|Lite Bot|Llaut|lycos|Mail\.RU_Bot|masscan|masidani_bot|Mediapartners-Google|Microsoft .{0,30} Bot|mogimogi|mozDex|MJ12bot|msnbot(?:-media {0,2}|)|msrbot|Mtps Feed Aggregation System|netresearch|Netvibes|NewsGator[^/]{0,30}|^NING|Nutch[^/]{0,30}|Nymesis|ObjectsSearch|OgScrper|Orbiter|OOZBOT|PagePeeker|PagesInventory|PaxleFramework|Peeplo Screenshot Bot|PHPCrawl|PlantyNet_WebRobot|Pompos|Qwantify|Read%20Later|Reaper|RedCarpet|Retreiver|Riddler|Rival IQ|scooter|Scrapy|Scrubby|searchsight|seekbot|semanticdiscovery|SemrushBot|Simpy|SimplePie|SEOstats|SimpleRSS|SiteCon|Slackbot-LinkExpanding|Slack-ImgProxy|Slurp|snappy|Speedy Spider|Squrl Java|Stringer|TheUsefulbot|ThumbShotsBot|Thumbshots\.ru|Tiny Tiny RSS|Twitterbot|WhatsApp|URL2PNG|Vagabondo|VoilaBot|^vortex|Votay bot|^voyager|WASALive.Bot|Web-sniffer|WebThumb|WeSEE:[A-z]{1,30}|WhatWeb|WIRE|WordPress|Wotbox|www\.almaden\.ibm\.com|Xenu(?:.s|) Link Sleuth|Xerka [A-z]{1,30}Bot|yacy(?:bot|)|YahooSeeker|Yahoo! Slurp|Yandex\w{1,30}|YodaoBot(?:-[A-z]{1,30}|)|YottaaMonitor|Yowedo|^Zao|^Zao-Crawler|ZeBot_www\.ze\.bz|ZooShot|ZyBorg|ArcGIS Hub Indexer|GPTBot)(?:[ /]v?(\d+)(?:\.(\d+)(?:\.(\d+)|)|)|)'
# AWS S3 Clients
# must come before "Bots General matcher" to catch "boto"/"boto3" before "bot"
- regex: '\b(Boto3?|JetS3t|aws-(?:cli|sdk-(?:cpp|go|java|nodejs|ruby2?|dotnet-(?:\d{1,2}|core)))|s3fs)/(\d+)\.(\d+)(?:\.(\d+)|)'
- regex: '\b(Boto3?|JetS3t|aws-(?:cli|sdk-(?:cpp|go|go-v\d|java|nodejs|ruby2?|dotnet-(?:\d{1,2}|core)))|s3fs)/(\d+)\.(\d+)(?:\.(\d+)|)'
# SAFE FME
- regex: '(FME)\/(\d+\.\d+)\.(\d+)\.(\d+)'
@@ -179,6 +179,9 @@ user_agent_parsers:
- regex: '\[FB.{0,300};'
family_replacement: 'Facebook'
# RecipeRadar crawler
- regex: '(RecipeRadar)/(\d+)\.(\d+)(?:\.(\d+)|)'
# Bots General matcher 'name/0.0'
- regex: '^.{0,200}?(?:\/[A-Za-z0-9\.]{0,50}|) {0,2}([A-Za-z0-9 \-_\!\[\]:]{0,50}(?:[Aa]rchiver|[Ii]ndexer|[Ss]craper|[Bb]ot|[Ss]pider|[Cc]rawl[a-z]{0,50}))[/ ](\d+)(?:\.(\d+)(?:\.(\d+)|)|)'
# Bots containing bot(but not CUBOT)
@@ -215,6 +218,16 @@ user_agent_parsers:
# Twitter
- regex: '(Twitter for (?:iPhone|iPad)|TwitterAndroid)(?:\/(\d+)\.(\d+)|)'
family_replacement: 'Twitter'
# TikTok
- regex: '(musical_ly) app_version\/(\d+)\.(\d+)\.(\d+)'
family_replacement: 'TikTok'
- regex: '(musical_ly_)(\d+)\.(\d+)\.(\d+)'
family_replacement: 'TikTok'
- regex: '(BytedanceWebview)\/[a-z0-9]+'
family_replacement: 'TikTok'
# KakaoTalk
- regex: 'Mozilla.{1,200}Mobile.{1,100}(KAKAOTALK)/(\d+)\.(\d+)\.(\d+)'
family_replacement: 'KakaoTalk'
# Phantom app
- regex: 'Mozilla.{1,200}Mobile.{1,100}(Phantom\/ios|Phantom\/android).(\d+)\.(\d+)\.(\d+)'
@@ -394,7 +407,7 @@ user_agent_parsers:
- regex: '(Instabridge)/(\d+)(?:\.(\d+)|)(?:\.(\d+)|)'
# Aloha Browser
- regex: '(AlohaBrowser)/(\d+)\.(\d+)\.(\d+)(?:\.(\d+)|)'
- regex: '(AlohaBrowser|ABB)/(\d+)\.(\d+)\.(\d+)(?:\.(\d+)|)'
family_replacement: 'Aloha Browser'
# Brave Browser https://brave.com/ , should go before Safari and Chrome Mobile
@@ -415,7 +428,7 @@ user_agent_parsers:
family_replacement: 'Edge Mobile'
# Oculus Browser, should go before Samsung Internet
- regex: '(OculusBrowser)/(\d+)\.(\d+).0.0(?:\.([0-9\-]+)|)'
- regex: '(OculusBrowser)/(\d+)\.(\d+)(?:\.([0-9\-]+)|)'
family_replacement: 'Oculus Browser'
# Samsung Internet (based on Chrome, but lacking some features)
@@ -492,6 +505,12 @@ user_agent_parsers:
- regex: '(Ecosia) android@(\d+)(?:\.(\d+)|)(?:\.(\d+)|)(?:\.(\d+)|)'
family_replacement: 'Ecosia Android'
# VivoBrowser
- regex: '(VivoBrowser)\/(\d+)\.(\d+)\.(\d+)\.(\d+)'
# HiBrowser
- regex: '(HiBrowser)\/v(\d+)\.(\d+)\.(\d+)\.(\d+)'
# Chrome Mobile
- regex: 'Version/.{1,300}(Chrome)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
family_replacement: 'Chrome Mobile WebView'
@@ -569,6 +588,85 @@ user_agent_parsers:
- regex: '^(surveyon)/(\d+)\.(\d+)\.(\d+)'
family_replacement: 'Surveyon'
# 115 Browser
- regex: '(115Browser)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
family_replacement: '115 Browser'
# Avira
- regex: '(Avira)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
family_replacement: 'Avira'
# CCleaner Browser
- regex: '(CCleaner)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
family_replacement: 'CCleaner'
# Norton
- regex: '(Norton)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
family_replacement: 'Norton'
# Quark
- regex: '(Quark)/(\d+)\.(\d+)\.(\d+)'
family_replacement: 'Quark'
# Quark PC
- regex: '(QuarkPC)/(\d+)\.(\d+)\.(\d+)'
family_replacement: 'Quark PC'
# Smart Lenovo Browser
- regex: '(SLBrowser)/(\d+)\.(\d+)\.(\d+)\.(\d+) SLBChan/(\d+)'
family_replacement: 'Smart Lenovo Browser'
# Atom Browser
- regex: '(Atom)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
family_replacement: 'Atom Browser'
# 360 Secure Browser
- regex: '(Chrome)/\d+\.\d+\.\d+\.\d+ .* QIHU 360(?:SEi18n|ENT)'
family_replacement: '360 Secure Browser'
# Decentr Web3 Browser
- regex: '(Decentr)'
family_replacement: 'Decentr Web3 Browser'
# Sparrow Browser
- regex: '(Sparrow)'
family_replacement: 'Sparrow Browser'
# Chromium GOST Browser
- regex: '(Chromium GOST)'
family_replacement: 'Chromium GOST Browser'
# AOL Shield Browser
- regex: '(AOLShield)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
family_replacement: 'AOL Shield Browser'
# Hola Browser
- regex: '(Hola)/(\d+)\.(\d+)\.(\d+)'
family_replacement: 'Hola Browser'
# Craving Explorer Browser
- regex: '(CravingExplorer)/(\d+)\.(\d+)\.(\d+)'
family_replacement: 'Craving Explorer Browser'
# Talon Cyber Security Browser
- regex: '(Talon)'
family_replacement: 'Talon Cyber Security Browser'
# QAX Browser
- regex: '(Qaxbrowser)'
family_replacement: 'QAX Browser'
# AOL Desktop Gold Browser
- regex: '(ADG)/(\d+)\.(\d+)\.(\d+)'
family_replacement: 'AOL Desktop Gold Browser'
# Sber Browser
- regex: '(SberBrowser)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
family_replacement: 'Sber Browser'
# JiSu Browser
- regex: '(JiSu)/(\d+)\.(\d+)\.(\d+)'
family_replacement: 'JiSu Browser'
#### END SPECIAL CASES TOP ####
#### MAIN CASES - this catches > 50% of all browsers ####
@@ -1387,6 +1485,13 @@ os_parsers:
# Box Drive and Box Sync on Mac OS X use OSX version numbers, not Darwin
- regex: '^Box.{0,200};(Darwin)/(10)\.(1\d)(?:\.(\d+)|)'
os_replacement: 'Mac OS X'
##########
# Hashicorp API
# APN/1.0 HashiCorp/1.0 Terraform/1.8.0 (+https://www.terraform.io) terraform-provider-aws/4.67.0 (+https://registry.terraform.io/providers/hashicorp/aws) aws-sdk-go/1.44.261 (go1.19.8; darwin; arm64)
##########
- regex: 'darwin; arm64'
os_replacement: 'Mac OS X'
##########
# iOS
@@ -1672,29 +1777,27 @@ os_parsers:
- regex: 'CFNetwork/.{0,100} Darwin/(21)\.\d+'
os_replacement: 'iOS'
os_v1_replacement: '15'
- regex: 'CFNetwork/.{0,100} Darwin/22\.0\.\d+'
- regex: 'CFNetwork/.{0,100} Darwin/22\.([0-5])\.\d+'
os_replacement: 'iOS'
os_v1_replacement: '16'
os_v2_replacement: '0'
- regex: 'CFNetwork/.{0,100} Darwin/22\.1\.\d+'
os_replacement: 'iOS'
os_v1_replacement: '16'
os_v2_replacement: '1'
- regex: 'CFNetwork/.{0,100} Darwin/22\.2\.\d+'
os_replacement: 'iOS'
os_v1_replacement: '16'
os_v2_replacement: '2'
- regex: 'CFNetwork/.{0,100} Darwin/22\.3\.\d+'
os_replacement: 'iOS'
os_v1_replacement: '16'
os_v2_replacement: '3'
- regex: 'CFNetwork/.{0,100} Darwin/22\.4\.\d+'
os_replacement: 'iOS'
os_v1_replacement: '16'
os_v2_replacement: '4'
os_v2_replacement: '$1'
- regex: 'CFNetwork/.{0,100} Darwin/(22)\.\d+'
os_replacement: 'iOS'
os_v1_replacement: '16'
- regex: 'CFNetwork/.{0,100} Darwin/23\.([0-5])\.\d+'
os_replacement: 'iOS'
os_v1_replacement: '17'
os_v2_replacement: '$1'
- regex: 'CFNetwork/.{0,100} Darwin/(23)\.\d+'
os_replacement: 'iOS'
os_v1_replacement: '17'
- regex: 'CFNetwork/.{0,100} Darwin/24\.([0-5])\.\d+'
os_replacement: 'iOS'
os_v1_replacement: '18'
os_v2_replacement: '$1'
- regex: 'CFNetwork/.{0,100} Darwin/(24)\.\d+'
os_replacement: 'iOS'
os_v1_replacement: '18'
- regex: 'CFNetwork/.{0,100} Darwin/'
os_replacement: 'iOS'
@@ -1889,6 +1992,21 @@ os_parsers:
# Roku Digital-Video-Players https://www.roku.com/
- regex: '^(Roku)/DVP-(\d+)\.(\d+)'
##########
# Amazon S3 client boto3
# Hasicorp API
# Boto3/1.28.62 md/Botocore#1.31.62 ua/2.0 os/macos#22.4.0 md/arch#arm64 lang/python#3.11.6 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.62
# APN/1.0 HashiCorp/1.0 Terraform/1.8.1 (+https://www.terraform.io) terraform-provider-aws/4.67.0 (+https://registry.terraform.io/providers/hashicorp/aws) aws-sdk-go-v2/1.18.0 os/macos lang/go/1.19.8 md/GOOS/darwin md/GOARCH/arm64 api/identitystore/1.16.11
##########
- regex: 'os\/macos[#]?(\d*)[.]?(\d*)[.]?(\d*)'
os_replacement: 'Mac OS X'
os_v1_replacement: '$1'
os_v2_replacement: '$2'
os_v3_replacement: '$3'
# Huawei HarmonyOS
- regex: '(HarmonyOS)[\s;]+(\d+|)\.?(\d+|)\.?(\d+|)'
device_parsers:
#########
@@ -5905,7 +6023,7 @@ device_parsers:
##########
# Spiders (this is a hack...)
##########
- regex: '^.{0,100}(bot|BUbiNG|zao|borg|DBot|oegp|silk|Xenu|zeal|^NING|CCBot|crawl|htdig|lycos|slurp|teoma|voila|yahoo|Sogou|CiBra|Nutch|^Java/|^JNLP/|Daumoa|Daum|Genieo|ichiro|larbin|pompos|Scrapy|snappy|speedy|spider|msnbot|msrbot|vortex|^vortex|crawler|favicon|indexer|Riddler|scooter|scraper|scrubby|WhatWeb|WinHTTP|bingbot|BingPreview|openbot|gigabot|furlbot|polybot|seekbot|^voyager|archiver|Icarus6j|mogimogi|Netvibes|blitzbot|altavista|charlotte|findlinks|Retreiver|TLSProber|WordPress|SeznamBot|ProoXiBot|wsr\-agent|Squrl Java|EtaoSpider|PaperLiBot|SputnikBot|A6\-Indexer|netresearch|searchsight|baiduspider|YisouSpider|ICC\-Crawler|http%20client|Python-urllib|dataparksearch|converacrawler|Screaming Frog|AppEngine-Google|YahooCacheSystem|fast\-webcrawler|Sogou Pic Spider|semanticdiscovery|Innovazion Crawler|facebookexternalhit|Google.{0,200}/\+/web/snippet|Google-HTTP-Java-Client|BlogBridge|IlTrovatore-Setaccio|InternetArchive|GomezAgent|WebThumbnail|heritrix|NewsGator|PagePeeker|Reaper|ZooShot|holmes|NL-Crawler|Pingdom|StatusCake|WhatsApp|masscan|Google Web Preview|Qwantify|Yeti|OgScrper)'
- regex: '^.{0,100}(bot|BUbiNG|zao|borg|DBot|oegp|silk|Xenu|zeal|^NING|CCBot|crawl|htdig|lycos|slurp|teoma|voila|yahoo|Sogou|CiBra|Nutch|^Java/|^JNLP/|Daumoa|Daum|Genieo|ichiro|larbin|pompos|Scrapy|snappy|speedy|spider|msnbot|msrbot|vortex|^vortex|crawler|favicon|indexer|Riddler|scooter|scraper|scrubby|WhatWeb|WinHTTP|bingbot|BingPreview|openbot|gigabot|furlbot|polybot|seekbot|^voyager|archiver|Icarus6j|mogimogi|Netvibes|blitzbot|altavista|charlotte|findlinks|Retreiver|TLSProber|WordPress|SeznamBot|ProoXiBot|wsr\-agent|Squrl Java|EtaoSpider|PaperLiBot|SputnikBot|A6\-Indexer|netresearch|searchsight|baiduspider|YisouSpider|ICC\-Crawler|http%20client|Python-urllib|dataparksearch|converacrawler|Screaming Frog|AppEngine-Google|YahooCacheSystem|fast\-webcrawler|Sogou Pic Spider|semanticdiscovery|Innovazion Crawler|facebookexternalhit|Google.{0,200}/\+/web/snippet|Google-HTTP-Java-Client|BlogBridge|IlTrovatore-Setaccio|InternetArchive|GomezAgent|WebThumbnail|heritrix|NewsGator|PagePeeker|Reaper|ZooShot|holmes|NL-Crawler|Pingdom|StatusCake|WhatsApp|masscan|Google Web Preview|Qwantify|Yeti|OgScrper|RecipeRadar|GPTBot)'
regex_flag: 'i'
device_replacement: 'Spider'
brand_replacement: 'Spider'
scripts/cli.cr
+12 -3
View File
@@ -19,11 +19,20 @@ OptionParser.parse do |parser|
exit
end
parser.on("--update-data", "Download all required data files (UA Parser and GeoLite2)") do
puts "=== Starting data files update ==="
App::Services::Cli.update_uap_regexes
App::Services::Cli.download_geolite_db
puts "=== All data files updated successfully ==="
exit
end
if ARGV.empty?
puts "Usage: ./cli [options]"
puts "Options:"
puts " --create-user=NAME Create a new user with the given name"
puts " --list-users List all users"
puts " --delete-user=USER_ID Delete a user by ID"
puts " --create-user=NAME Create a new user with the given name"
puts " --list-users List all users"
puts " --delete-user=USER_ID Delete a user by ID"
puts " --update-data Download all required data files"
end
end