diff --git a/app/controllers/link.cr b/app/controllers/link.cr index c70f0fe..62ab5cd 100644 --- a/app/controllers/link.cr +++ b/app/controllers/link.cr @@ -4,7 +4,7 @@ require "user_agent_parser" require "../lib/controller.cr" require "../lib/ip_lookup" -UserAgent.load_regexes(File.read("data/regexes.yaml")) +UserAgent.load_regexes(File.read("data/uap_core_regexes.yaml")) IpLookup.load_mmdb("data/GeoLite2-Country.mmdb") module App::Controllers::Link diff --git a/app/services/cli.cr b/app/services/cli.cr index 0c28b5e..305184c 100644 --- a/app/services/cli.cr +++ b/app/services/cli.cr @@ -1,3 +1,6 @@ +require "file_utils" +require "http/client" + require "../config/*" require "../lib/*" require "../models/*" @@ -53,4 +56,86 @@ module App::Services::Cli puts "Admin setup skipped: Missing ADMIN_NAME or ADMIN_API_KEY environment variables." end end + + def self.update_uap_regexes + puts "Downloading User-Agent Parser core regexes..." + + FileUtils.mkdir_p("data") + url = "https://raw.githubusercontent.com/ua-parser/uap-core/master/regexes.yaml" + output_file = "data/uap_core_regexes.yaml" + + begin + http_get_with_redirect(url) do |response| + File.write(output_file, response.body_io.gets_to_end) + end + puts "User-Agent regexes downloaded to #{output_file}" + rescue e + puts "Error: Failed to download UAP core regexes: #{e.message}" + end + end + + def self.download_geolite_db + puts "Downloading GeoLite2 Country database..." + + FileUtils.mkdir_p("data") + url = "https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-Country.mmdb" + output_file = "data/GeoLite2-Country.mmdb" + + begin + File.open(output_file, "wb") do |file| + http_get_with_redirect(url) do |response| + IO.copy(response.body_io, file) + end + end + puts "GeoLite2 database downloaded to #{output_file}" + rescue e + puts "Error: Failed to download GeoLite2 database: #{e.message}" + end + end + + private def self.http_get_with_redirect(url : String, max_redirects = 5) + redirects = 0 + + while redirects < max_redirects + uri = URI.parse(url) + client = HTTP::Client.new(uri) + + success = false + follow_redirect = false + redirect_url = nil + + begin + client.get(uri.request_target) do |response| + case response.status_code + when 200 + yield response + success = true + when 301, 302 + if new_location = response.headers["Location"]? + puts "Following redirect to: #{new_location}" + redirect_url = new_location + follow_redirect = true + else + raise "Received redirect status but no Location header" + end + else + raise "Failed request with status code: #{response.status_code}" + end + end + ensure + client.close + end + + return if success + + if follow_redirect && redirect_url + url = redirect_url + redirects += 1 + else + break + end + end + + raise "Too many redirects (#{max_redirects})" + end end diff --git a/data/regexes.yaml b/data/uap_core_regexes.yaml similarity index 97% rename from data/regexes.yaml rename to data/uap_core_regexes.yaml index 856fbc1..26a64f1 100644 --- a/data/regexes.yaml +++ b/data/uap_core_regexes.yaml @@ -148,11 +148,11 @@ user_agent_parsers: family_replacement: 'Pinterestbot' # Bots - - regex: '(CSimpleSpider|Cityreview Robot|CrawlDaddy|CrawlFire|Finderbots|Index crawler|Job Roboter|KiwiStatus Spider|Lijit Crawler|QuerySeekerSpider|ScollSpider|Trends Crawler|USyd-NLP-Spider|SiteCat Webbot|BotName\/\$BotVersion|123metaspider-Bot|1470\.net crawler|50\.nu|8bo Crawler Bot|Aboundex|Accoona-[A-z]{1,30}-Agent|AdsBot-Google(?:-[a-z]{1,30}|)|altavista|AppEngine-Google|archive.{0,30}\.org_bot|archiver|Ask Jeeves|[Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]{1,30})(?:-[A-Za-z]{1,30}|)|bingbot|BingPreview|blitzbot|BlogBridge|Bloglovin|BoardReader Blog Indexer|BoardReader Favicon Fetcher|boitho.com-dc|BotSeer|BUbiNG|\b\w{0,30}favicon\w{0,30}\b|\bYeti(?:-[a-z]{1,30}|)|Catchpoint(?: bot|)|[Cc]harlotte|Checklinks|clumboot|Comodo HTTP\(S\) Crawler|Comodo-Webinspector-Crawler|ConveraCrawler|CRAWL-E|CrawlConvera|Daumoa(?:-feedfetcher|)|Feed Seeker Bot|Feedbin|findlinks|Flamingo_SearchEngine|FollowSite Bot|furlbot|Genieo|gigabot|GomezAgent|gonzo1|(?:[a-zA-Z]{1,30}-|)Googlebot(?:-[a-zA-Z]{1,30}|)|Google SketchUp|grub-client|gsa-crawler|heritrix|HiddenMarket|holmes|HooWWWer|htdig|ia_archiver|ICC-Crawler|Icarus6j|ichiro(?:/mobile|)|IconSurf|IlTrovatore(?:-Setaccio|)|InfuzApp|Innovazion Crawler|InternetArchive|IP2[a-z]{1,30}Bot|jbot\b|KaloogaBot|Kraken|Kurzor|larbin|LEIA|LesnikBot|Linguee Bot|LinkAider|LinkedInBot|Lite Bot|Llaut|lycos|Mail\.RU_Bot|masscan|masidani_bot|Mediapartners-Google|Microsoft .{0,30} Bot|mogimogi|mozDex|MJ12bot|msnbot(?:-media {0,2}|)|msrbot|Mtps Feed Aggregation System|netresearch|Netvibes|NewsGator[^/]{0,30}|^NING|Nutch[^/]{0,30}|Nymesis|ObjectsSearch|OgScrper|Orbiter|OOZBOT|PagePeeker|PagesInventory|PaxleFramework|Peeplo Screenshot Bot|PHPCrawl|PlantyNet_WebRobot|Pompos|Qwantify|Read%20Later|Reaper|RedCarpet|Retreiver|Riddler|Rival IQ|scooter|Scrapy|Scrubby|searchsight|seekbot|semanticdiscovery|SemrushBot|Simpy|SimplePie|SEOstats|SimpleRSS|SiteCon|Slackbot-LinkExpanding|Slack-ImgProxy|Slurp|snappy|Speedy Spider|Squrl Java|Stringer|TheUsefulbot|ThumbShotsBot|Thumbshots\.ru|Tiny Tiny RSS|Twitterbot|WhatsApp|URL2PNG|Vagabondo|VoilaBot|^vortex|Votay bot|^voyager|WASALive.Bot|Web-sniffer|WebThumb|WeSEE:[A-z]{1,30}|WhatWeb|WIRE|WordPress|Wotbox|www\.almaden\.ibm\.com|Xenu(?:.s|) Link Sleuth|Xerka [A-z]{1,30}Bot|yacy(?:bot|)|YahooSeeker|Yahoo! Slurp|Yandex\w{1,30}|YodaoBot(?:-[A-z]{1,30}|)|YottaaMonitor|Yowedo|^Zao|^Zao-Crawler|ZeBot_www\.ze\.bz|ZooShot|ZyBorg|ArcGIS Hub Indexer)(?:[ /]v?(\d+)(?:\.(\d+)(?:\.(\d+)|)|)|)' + - regex: '(CSimpleSpider|Cityreview Robot|CrawlDaddy|CrawlFire|Finderbots|Index crawler|Job Roboter|KiwiStatus Spider|Lijit Crawler|QuerySeekerSpider|ScollSpider|Trends Crawler|USyd-NLP-Spider|SiteCat Webbot|BotName\/\$BotVersion|123metaspider-Bot|1470\.net crawler|50\.nu|8bo Crawler Bot|Aboundex|Accoona-[A-z]{1,30}-Agent|AdsBot-Google(?:-[a-z]{1,30}|)|altavista|AppEngine-Google|archive.{0,30}\.org_bot|archiver|Ask Jeeves|[Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]{1,30})(?:-[A-Za-z]{1,30}|)|bingbot|BingPreview|blitzbot|BlogBridge|Bloglovin|BoardReader Blog Indexer|BoardReader Favicon Fetcher|boitho.com-dc|BotSeer|BUbiNG|\b\w{0,30}favicon\w{0,30}\b|\bYeti(?:-[a-z]{1,30}|)|Catchpoint(?: bot|)|[Cc]harlotte|Checklinks|clumboot|Comodo HTTP\(S\) Crawler|Comodo-Webinspector-Crawler|ConveraCrawler|CRAWL-E|CrawlConvera|Daumoa(?:-feedfetcher|)|Feed Seeker Bot|Feedbin|findlinks|Flamingo_SearchEngine|FollowSite Bot|furlbot|Genieo|gigabot|GomezAgent|gonzo1|(?:[a-zA-Z]{1,30}-|)Googlebot(?:-[a-zA-Z]{1,30}|)|GoogleOther|Google SketchUp|grub-client|gsa-crawler|heritrix|HiddenMarket|holmes|HooWWWer|htdig|ia_archiver|ICC-Crawler|Icarus6j|ichiro(?:/mobile|)|IconSurf|IlTrovatore(?:-Setaccio|)|InfuzApp|Innovazion Crawler|InternetArchive|IP2[a-z]{1,30}Bot|jbot\b|KaloogaBot|Kraken|Kurzor|larbin|LEIA|LesnikBot|Linguee Bot|LinkAider|LinkedInBot|Lite Bot|Llaut|lycos|Mail\.RU_Bot|masscan|masidani_bot|Mediapartners-Google|Microsoft .{0,30} Bot|mogimogi|mozDex|MJ12bot|msnbot(?:-media {0,2}|)|msrbot|Mtps Feed Aggregation System|netresearch|Netvibes|NewsGator[^/]{0,30}|^NING|Nutch[^/]{0,30}|Nymesis|ObjectsSearch|OgScrper|Orbiter|OOZBOT|PagePeeker|PagesInventory|PaxleFramework|Peeplo Screenshot Bot|PHPCrawl|PlantyNet_WebRobot|Pompos|Qwantify|Read%20Later|Reaper|RedCarpet|Retreiver|Riddler|Rival IQ|scooter|Scrapy|Scrubby|searchsight|seekbot|semanticdiscovery|SemrushBot|Simpy|SimplePie|SEOstats|SimpleRSS|SiteCon|Slackbot-LinkExpanding|Slack-ImgProxy|Slurp|snappy|Speedy Spider|Squrl Java|Stringer|TheUsefulbot|ThumbShotsBot|Thumbshots\.ru|Tiny Tiny RSS|Twitterbot|WhatsApp|URL2PNG|Vagabondo|VoilaBot|^vortex|Votay bot|^voyager|WASALive.Bot|Web-sniffer|WebThumb|WeSEE:[A-z]{1,30}|WhatWeb|WIRE|WordPress|Wotbox|www\.almaden\.ibm\.com|Xenu(?:.s|) Link Sleuth|Xerka [A-z]{1,30}Bot|yacy(?:bot|)|YahooSeeker|Yahoo! Slurp|Yandex\w{1,30}|YodaoBot(?:-[A-z]{1,30}|)|YottaaMonitor|Yowedo|^Zao|^Zao-Crawler|ZeBot_www\.ze\.bz|ZooShot|ZyBorg|ArcGIS Hub Indexer|GPTBot)(?:[ /]v?(\d+)(?:\.(\d+)(?:\.(\d+)|)|)|)' # AWS S3 Clients # must come before "Bots General matcher" to catch "boto"/"boto3" before "bot" - - regex: '\b(Boto3?|JetS3t|aws-(?:cli|sdk-(?:cpp|go|java|nodejs|ruby2?|dotnet-(?:\d{1,2}|core)))|s3fs)/(\d+)\.(\d+)(?:\.(\d+)|)' + - regex: '\b(Boto3?|JetS3t|aws-(?:cli|sdk-(?:cpp|go|go-v\d|java|nodejs|ruby2?|dotnet-(?:\d{1,2}|core)))|s3fs)/(\d+)\.(\d+)(?:\.(\d+)|)' # SAFE FME - regex: '(FME)\/(\d+\.\d+)\.(\d+)\.(\d+)' @@ -179,6 +179,9 @@ user_agent_parsers: - regex: '\[FB.{0,300};' family_replacement: 'Facebook' + # RecipeRadar crawler + - regex: '(RecipeRadar)/(\d+)\.(\d+)(?:\.(\d+)|)' + # Bots General matcher 'name/0.0' - regex: '^.{0,200}?(?:\/[A-Za-z0-9\.]{0,50}|) {0,2}([A-Za-z0-9 \-_\!\[\]:]{0,50}(?:[Aa]rchiver|[Ii]ndexer|[Ss]craper|[Bb]ot|[Ss]pider|[Cc]rawl[a-z]{0,50}))[/ ](\d+)(?:\.(\d+)(?:\.(\d+)|)|)' # Bots containing bot(but not CUBOT) @@ -215,6 +218,16 @@ user_agent_parsers: # Twitter - regex: '(Twitter for (?:iPhone|iPad)|TwitterAndroid)(?:\/(\d+)\.(\d+)|)' family_replacement: 'Twitter' + # TikTok + - regex: '(musical_ly) app_version\/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'TikTok' + - regex: '(musical_ly_)(\d+)\.(\d+)\.(\d+)' + family_replacement: 'TikTok' + - regex: '(BytedanceWebview)\/[a-z0-9]+' + family_replacement: 'TikTok' + # KakaoTalk + - regex: 'Mozilla.{1,200}Mobile.{1,100}(KAKAOTALK)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'KakaoTalk' # Phantom app - regex: 'Mozilla.{1,200}Mobile.{1,100}(Phantom\/ios|Phantom\/android).(\d+)\.(\d+)\.(\d+)' @@ -394,7 +407,7 @@ user_agent_parsers: - regex: '(Instabridge)/(\d+)(?:\.(\d+)|)(?:\.(\d+)|)' # Aloha Browser - - regex: '(AlohaBrowser)/(\d+)\.(\d+)\.(\d+)(?:\.(\d+)|)' + - regex: '(AlohaBrowser|ABB)/(\d+)\.(\d+)\.(\d+)(?:\.(\d+)|)' family_replacement: 'Aloha Browser' # Brave Browser https://brave.com/ , should go before Safari and Chrome Mobile @@ -415,7 +428,7 @@ user_agent_parsers: family_replacement: 'Edge Mobile' # Oculus Browser, should go before Samsung Internet - - regex: '(OculusBrowser)/(\d+)\.(\d+).0.0(?:\.([0-9\-]+)|)' + - regex: '(OculusBrowser)/(\d+)\.(\d+)(?:\.([0-9\-]+)|)' family_replacement: 'Oculus Browser' # Samsung Internet (based on Chrome, but lacking some features) @@ -492,6 +505,12 @@ user_agent_parsers: - regex: '(Ecosia) android@(\d+)(?:\.(\d+)|)(?:\.(\d+)|)(?:\.(\d+)|)' family_replacement: 'Ecosia Android' + # VivoBrowser + - regex: '(VivoBrowser)\/(\d+)\.(\d+)\.(\d+)\.(\d+)' + + # HiBrowser + - regex: '(HiBrowser)\/v(\d+)\.(\d+)\.(\d+)\.(\d+)' + # Chrome Mobile - regex: 'Version/.{1,300}(Chrome)/(\d+)\.(\d+)\.(\d+)\.(\d+)' family_replacement: 'Chrome Mobile WebView' @@ -569,6 +588,85 @@ user_agent_parsers: - regex: '^(surveyon)/(\d+)\.(\d+)\.(\d+)' family_replacement: 'Surveyon' + # 115 Browser + - regex: '(115Browser)/(\d+)\.(\d+)\.(\d+)\.(\d+)' + family_replacement: '115 Browser' + + # Avira + - regex: '(Avira)/(\d+)\.(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Avira' + + # CCleaner Browser + - regex: '(CCleaner)/(\d+)\.(\d+)\.(\d+)\.(\d+)' + family_replacement: 'CCleaner' + + # Norton + - regex: '(Norton)/(\d+)\.(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Norton' + + # Quark + - regex: '(Quark)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Quark' + # Quark PC + - regex: '(QuarkPC)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Quark PC' + + # Smart Lenovo Browser + - regex: '(SLBrowser)/(\d+)\.(\d+)\.(\d+)\.(\d+) SLBChan/(\d+)' + family_replacement: 'Smart Lenovo Browser' + + # Atom Browser + - regex: '(Atom)/(\d+)\.(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Atom Browser' + + # 360 Secure Browser + - regex: '(Chrome)/\d+\.\d+\.\d+\.\d+ .* QIHU 360(?:SEi18n|ENT)' + family_replacement: '360 Secure Browser' + + # Decentr Web3 Browser + - regex: '(Decentr)' + family_replacement: 'Decentr Web3 Browser' + + # Sparrow Browser + - regex: '(Sparrow)' + family_replacement: 'Sparrow Browser' + + # Chromium GOST Browser + - regex: '(Chromium GOST)' + family_replacement: 'Chromium GOST Browser' + + # AOL Shield Browser + - regex: '(AOLShield)/(\d+)\.(\d+)\.(\d+)\.(\d+)' + family_replacement: 'AOL Shield Browser' + + # Hola Browser + - regex: '(Hola)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Hola Browser' + + # Craving Explorer Browser + - regex: '(CravingExplorer)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Craving Explorer Browser' + + # Talon Cyber Security Browser + - regex: '(Talon)' + family_replacement: 'Talon Cyber Security Browser' + + # QAX Browser + - regex: '(Qaxbrowser)' + family_replacement: 'QAX Browser' + + # AOL Desktop Gold Browser + - regex: '(ADG)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'AOL Desktop Gold Browser' + + # Sber Browser + - regex: '(SberBrowser)/(\d+)\.(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Sber Browser' + + # JiSu Browser + - regex: '(JiSu)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'JiSu Browser' + #### END SPECIAL CASES TOP #### #### MAIN CASES - this catches > 50% of all browsers #### @@ -1387,6 +1485,13 @@ os_parsers: # Box Drive and Box Sync on Mac OS X use OSX version numbers, not Darwin - regex: '^Box.{0,200};(Darwin)/(10)\.(1\d)(?:\.(\d+)|)' os_replacement: 'Mac OS X' + + ########## + # Hashicorp API + # APN/1.0 HashiCorp/1.0 Terraform/1.8.0 (+https://www.terraform.io) terraform-provider-aws/4.67.0 (+https://registry.terraform.io/providers/hashicorp/aws) aws-sdk-go/1.44.261 (go1.19.8; darwin; arm64) + ########## + - regex: 'darwin; arm64' + os_replacement: 'Mac OS X' ########## # iOS @@ -1672,29 +1777,27 @@ os_parsers: - regex: 'CFNetwork/.{0,100} Darwin/(21)\.\d+' os_replacement: 'iOS' os_v1_replacement: '15' - - regex: 'CFNetwork/.{0,100} Darwin/22\.0\.\d+' + - regex: 'CFNetwork/.{0,100} Darwin/22\.([0-5])\.\d+' os_replacement: 'iOS' os_v1_replacement: '16' - os_v2_replacement: '0' - - regex: 'CFNetwork/.{0,100} Darwin/22\.1\.\d+' - os_replacement: 'iOS' - os_v1_replacement: '16' - os_v2_replacement: '1' - - regex: 'CFNetwork/.{0,100} Darwin/22\.2\.\d+' - os_replacement: 'iOS' - os_v1_replacement: '16' - os_v2_replacement: '2' - - regex: 'CFNetwork/.{0,100} Darwin/22\.3\.\d+' - os_replacement: 'iOS' - os_v1_replacement: '16' - os_v2_replacement: '3' - - regex: 'CFNetwork/.{0,100} Darwin/22\.4\.\d+' - os_replacement: 'iOS' - os_v1_replacement: '16' - os_v2_replacement: '4' + os_v2_replacement: '$1' - regex: 'CFNetwork/.{0,100} Darwin/(22)\.\d+' os_replacement: 'iOS' os_v1_replacement: '16' + - regex: 'CFNetwork/.{0,100} Darwin/23\.([0-5])\.\d+' + os_replacement: 'iOS' + os_v1_replacement: '17' + os_v2_replacement: '$1' + - regex: 'CFNetwork/.{0,100} Darwin/(23)\.\d+' + os_replacement: 'iOS' + os_v1_replacement: '17' + - regex: 'CFNetwork/.{0,100} Darwin/24\.([0-5])\.\d+' + os_replacement: 'iOS' + os_v1_replacement: '18' + os_v2_replacement: '$1' + - regex: 'CFNetwork/.{0,100} Darwin/(24)\.\d+' + os_replacement: 'iOS' + os_v1_replacement: '18' - regex: 'CFNetwork/.{0,100} Darwin/' os_replacement: 'iOS' @@ -1889,6 +1992,21 @@ os_parsers: # Roku Digital-Video-Players https://www.roku.com/ - regex: '^(Roku)/DVP-(\d+)\.(\d+)' + ########## + # Amazon S3 client boto3 + # Hasicorp API + # Boto3/1.28.62 md/Botocore#1.31.62 ua/2.0 os/macos#22.4.0 md/arch#arm64 lang/python#3.11.6 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.62 + # APN/1.0 HashiCorp/1.0 Terraform/1.8.1 (+https://www.terraform.io) terraform-provider-aws/4.67.0 (+https://registry.terraform.io/providers/hashicorp/aws) aws-sdk-go-v2/1.18.0 os/macos lang/go/1.19.8 md/GOOS/darwin md/GOARCH/arm64 api/identitystore/1.16.11 + ########## + - regex: 'os\/macos[#]?(\d*)[.]?(\d*)[.]?(\d*)' + os_replacement: 'Mac OS X' + os_v1_replacement: '$1' + os_v2_replacement: '$2' + os_v3_replacement: '$3' + + # Huawei HarmonyOS + - regex: '(HarmonyOS)[\s;]+(\d+|)\.?(\d+|)\.?(\d+|)' + device_parsers: ######### @@ -5905,7 +6023,7 @@ device_parsers: ########## # Spiders (this is a hack...) ########## - - regex: '^.{0,100}(bot|BUbiNG|zao|borg|DBot|oegp|silk|Xenu|zeal|^NING|CCBot|crawl|htdig|lycos|slurp|teoma|voila|yahoo|Sogou|CiBra|Nutch|^Java/|^JNLP/|Daumoa|Daum|Genieo|ichiro|larbin|pompos|Scrapy|snappy|speedy|spider|msnbot|msrbot|vortex|^vortex|crawler|favicon|indexer|Riddler|scooter|scraper|scrubby|WhatWeb|WinHTTP|bingbot|BingPreview|openbot|gigabot|furlbot|polybot|seekbot|^voyager|archiver|Icarus6j|mogimogi|Netvibes|blitzbot|altavista|charlotte|findlinks|Retreiver|TLSProber|WordPress|SeznamBot|ProoXiBot|wsr\-agent|Squrl Java|EtaoSpider|PaperLiBot|SputnikBot|A6\-Indexer|netresearch|searchsight|baiduspider|YisouSpider|ICC\-Crawler|http%20client|Python-urllib|dataparksearch|converacrawler|Screaming Frog|AppEngine-Google|YahooCacheSystem|fast\-webcrawler|Sogou Pic Spider|semanticdiscovery|Innovazion Crawler|facebookexternalhit|Google.{0,200}/\+/web/snippet|Google-HTTP-Java-Client|BlogBridge|IlTrovatore-Setaccio|InternetArchive|GomezAgent|WebThumbnail|heritrix|NewsGator|PagePeeker|Reaper|ZooShot|holmes|NL-Crawler|Pingdom|StatusCake|WhatsApp|masscan|Google Web Preview|Qwantify|Yeti|OgScrper)' + - regex: '^.{0,100}(bot|BUbiNG|zao|borg|DBot|oegp|silk|Xenu|zeal|^NING|CCBot|crawl|htdig|lycos|slurp|teoma|voila|yahoo|Sogou|CiBra|Nutch|^Java/|^JNLP/|Daumoa|Daum|Genieo|ichiro|larbin|pompos|Scrapy|snappy|speedy|spider|msnbot|msrbot|vortex|^vortex|crawler|favicon|indexer|Riddler|scooter|scraper|scrubby|WhatWeb|WinHTTP|bingbot|BingPreview|openbot|gigabot|furlbot|polybot|seekbot|^voyager|archiver|Icarus6j|mogimogi|Netvibes|blitzbot|altavista|charlotte|findlinks|Retreiver|TLSProber|WordPress|SeznamBot|ProoXiBot|wsr\-agent|Squrl Java|EtaoSpider|PaperLiBot|SputnikBot|A6\-Indexer|netresearch|searchsight|baiduspider|YisouSpider|ICC\-Crawler|http%20client|Python-urllib|dataparksearch|converacrawler|Screaming Frog|AppEngine-Google|YahooCacheSystem|fast\-webcrawler|Sogou Pic Spider|semanticdiscovery|Innovazion Crawler|facebookexternalhit|Google.{0,200}/\+/web/snippet|Google-HTTP-Java-Client|BlogBridge|IlTrovatore-Setaccio|InternetArchive|GomezAgent|WebThumbnail|heritrix|NewsGator|PagePeeker|Reaper|ZooShot|holmes|NL-Crawler|Pingdom|StatusCake|WhatsApp|masscan|Google Web Preview|Qwantify|Yeti|OgScrper|RecipeRadar|GPTBot)' regex_flag: 'i' device_replacement: 'Spider' brand_replacement: 'Spider' diff --git a/scripts/cli.cr b/scripts/cli.cr index 9f7e9bb..f0024d9 100644 --- a/scripts/cli.cr +++ b/scripts/cli.cr @@ -19,11 +19,20 @@ OptionParser.parse do |parser| exit end + parser.on("--update-data", "Download all required data files (UA Parser and GeoLite2)") do + puts "=== Starting data files update ===" + App::Services::Cli.update_uap_regexes + App::Services::Cli.download_geolite_db + puts "=== All data files updated successfully ===" + exit + end + if ARGV.empty? puts "Usage: ./cli [options]" puts "Options:" - puts " --create-user=NAME Create a new user with the given name" - puts " --list-users List all users" - puts " --delete-user=USER_ID Delete a user by ID" + puts " --create-user=NAME Create a new user with the given name" + puts " --list-users List all users" + puts " --delete-user=USER_ID Delete a user by ID" + puts " --update-data Download all required data files" end end