Merge pull request #16 from sjdonado/automated/update-parsers

chore: Update UA regexes and GeoLite2 database

.dockerignore

@@ -1,9 +1,44 @@
 .git
-/bin/
+.gitignore
-/.shards/
+.github
-/spec/
-/sqlite/
 
+/bin/
+/bit
+/cli
+/benchmark
+*.dwarf
+*.o
+*.a
+
+# Dependencies cache
+/.shards/
+/lib/.shards/
+
+/spec/
+
+# Database files (should be mounted as volumes)
+/sqlite/
+*.db
+*.db-shm
+*.db-wal
+
+# Logs and temporary files
+*.log
+
+# Documentation
 /docs/
-benchmark.cr
+*.md
+README.md
+CODE_OF_CONDUCT.md
+CONTRIBUTING.md
+LICENSE
+DOCKER_MIGRATION.md
+
+# Development environment
 .env*
+.editorconfig
+
+# Docker files (not needed inside image)
+Dockerfile
+docker-compose.yml
+.dockerignore

.gitignore

@@ -9,3 +9,4 @@
 
 *.log
 bit
+cli

README.md

@@ -1,5 +1,4 @@
 [![Docker Pulls](https://img.shields.io/docker/pulls/sjdonado/bit.svg)](https://hub.docker.com/r/sjdonado/bit)
-[![Docker Stars](https://img.shields.io/docker/stars/sjdonado/bit.svg)](https://hub.docker.com/r/sjdonado/bit)
 [![Docker Image Size](https://img.shields.io/docker/image-size/sjdonado/bit/latest)](https://hub.docker.com/r/sjdonado/bit)
 
 ## Features
@@ -8,7 +7,7 @@
 - Includes `X-Forwarded-For` header.
 - Multiple users are supported via API key authentication. Create, list and delete keys via the [CLI](docs/SETUP.md#cli).
 - Easy to extend, Ruby on Rails inspired setup.
-- Auto update UA regexes and GeoLite2 database
+- Auto update UA regexes and GeoLite2 database.
 
 ## Why bit?
 
@@ -22,7 +21,7 @@
 
 ## Run It Anywhere
 
-Images available on [Docker Hub](https://hub.docker.com/r/sjdonado/bit/tags).
+All images available on [Docker Hub](https://hub.docker.com/r/sjdonado/bit/tags).
 
 ### Docker
 

data/uap_core_regexes.yaml

@@ -93,6 +93,10 @@ user_agent_parsers:
   - regex: '(NewRelicPinger)/(\d+)\.(\d+)'
     family_replacement: 'NewRelicPingerBot'
 
+  # Dynatrace/Ruxit synthetic monitor
+  - regex: '(RuxitSynthetic)/(\d+)\.(\d+)'
+    family_replacement: 'Ruxit Synthetic'
+
   # Tableau
   - regex: '(Tableau)/(\d+)\.(\d+)'
     family_replacement: 'Tableau'
@@ -206,7 +210,12 @@ user_agent_parsers:
   - regex: '\[(Pinterest)/[^\]]{1,50}\]'
   - regex: '(Pinterest)(?: for Android(?: Tablet|)|)/(\d+)(?:\.(\d+)|)(?:\.(\d+)|)'
   # Instagram app
+  # iOS Instagram embeds the token inside a full WebKit UA:
+  #   Mozilla/5.0 (iPhone; ...) Mobile/... Instagram VERSION (...)
+  # Android Instagram uses a bare format with no browser wrapper:
+  #   Instagram VERSION Android (...)
   - regex: 'Mozilla.{1,200}Mobile.{1,100}(Instagram).(\d+)\.(\d+)\.(\d+)'
+  - regex: '(Instagram) (\d+)\.(\d+)\.(\d+)'
   # Flipboard app
   - regex: 'Mozilla.{1,200}Mobile.{1,100}(Flipboard).(\d+)\.(\d+)\.(\d+)'
   # Flipboard-briefing app
@@ -228,6 +237,9 @@ user_agent_parsers:
   # KakaoTalk
   - regex: 'Mozilla.{1,200}Mobile.{1,100}(KAKAOTALK)/(\d+)\.(\d+)\.(\d+)'
     family_replacement: 'KakaoTalk'
+  # Telegram
+  - regex: '(Telegram-Android)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Telegram'
 
   # Phantom app
   - regex: 'Mozilla.{1,200}Mobile.{1,100}(Phantom\/ios|Phantom\/android).(\d+)\.(\d+)\.(\d+)'
@@ -248,6 +260,10 @@ user_agent_parsers:
   - regex: '(PaleMoon)/(\d+)\.(\d+)(?:\.(\d+)|)'
     family_replacement: 'Pale Moon'
 
+  # Camoufox - anti-detect Firefox fork for web scraping/automation; replaces the
+  # Firefox version token with "Camoufox Camoufox VERSION" in the UA string
+  - regex: '(Camoufox) Camoufox (\d+)\.(\d+)'
+
   # Firefox
   - regex: '(Fennec)/(\d+)\.(\d+)\.?([ab]?\d+[a-z]*)'
     family_replacement: 'Firefox Mobile'
@@ -296,7 +312,7 @@ user_agent_parsers:
 
   # UC Browser
   # we need check it before opera. In other case case UC Browser detected look like Opera Mini
-  - regex: '(UC? ?Browser|UCWEB|U3)[ /]?(\d+)\.(\d+)\.(\d+)'
+  - regex: '(UC? ?Browser|UCWEB|UCMobile|U3)[ /]?(\d+)\.(\d+)\.(\d+)'
     family_replacement: 'UC Browser'
 
   # Opera will stop at 9.80 and hide the real version in the Version string.
@@ -321,6 +337,14 @@ user_agent_parsers:
   - regex: '(?:Chrome).{1,300}(OPR)/(\d+)\.(\d+)\.(\d+)'
     family_replacement: 'Opera'
 
+  # Opera GX uses "OPX" instead of "OPR"
+  - regex: '(OPX)/(\d+)\.(\d+)(?:\.(\d+)|)'
+    family_replacement: 'Opera GX'
+
+  # Opera Touch uses "OPT"
+  - regex: '(OPT)/(\d+)\.(\d+)(?:\.(\d+)|)'
+    family_replacement: 'Opera Touch'
+
   # Opera Coast
   - regex: '(Coast)/(\d+).(\d+).(\d+)'
     family_replacement: 'Opera Coast'
@@ -410,10 +434,14 @@ user_agent_parsers:
   - regex: '(AlohaBrowser|ABB)/(\d+)\.(\d+)\.(\d+)(?:\.(\d+)|)'
     family_replacement: 'Aloha Browser'
 
-  # Brave Browser https://brave.com/ , should go before Safari and Chrome Mobile
+  # Brave Browser, should go before Safari and Chrome Mobile
   - regex: '((?:B|b)rave(?:\sChrome)?)/(\d+)(?:\.(\d+)|)(?:\.(\d+)|)(?:\.(\d+)|)'
     family_replacement: 'Brave'
 
+  # Brave iOS Browser, checks for (Brave) or Brave at end
+  - regex: '(?:\()?Brave(?:\))?\s*$'
+    family_replacement: 'Brave'
+
   # Amazon Silk, should go before Safari and Chrome Mobile
   - regex: '(Silk)/(\d+)\.(\d+)(?:\.([0-9\-]+)|)'
     family_replacement: 'Amazon Silk'
@@ -509,7 +537,20 @@ user_agent_parsers:
   - regex: '(VivoBrowser)\/(\d+)\.(\d+)\.(\d+)(?:\.(\d+)|)'
 
   # HiBrowser
-  - regex: '(HiBrowser)\/v(\d+)\.(\d+)\.(\d+)\.(\d+)'
+  - regex: '(H[Ii]Browser)\/v(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'HiBrowser'
+
+  # Honor Browser
+  - regex: '(HonorBrowser)/(\d+)\.(\d+)\.(\d+)(?:\.(\d+)|)'
+    family_replacement: 'Honor Browser'
+
+  # Honor Browser
+  - regex: '(bdhonorbrowser)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Honor Browser'
+
+  # HeyTap Browser
+  - regex: '(HeyTapBrowser)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'HeyTap Browser'
 
   # Weibo
   # Must before Chrome Mobile WebView
@@ -518,6 +559,10 @@ user_agent_parsers:
   - regex: '(WeiboliteiOS|WeiboIntliOS)'
     family_replacement: 'Weibo'
 
+  # Phoenix Browser
+  - regex: '(PHX)/(\d+)\.(\d+)'
+    family_replacement: 'Phoenix Browser'
+
   # Chrome Mobile
   - regex: 'Version/.{1,300}(Chrome)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
     family_replacement: 'Chrome Mobile WebView'
@@ -619,7 +664,7 @@ user_agent_parsers:
     family_replacement: 'Quark PC'
 
   # Smart Lenovo Browser
-  - regex: '(SLBrowser)/(\d+)\.(\d+)\.(\d+)\.(\d+) SLBChan/(\d+)'
+  - regex: '(SLBrowser)/(\d+)\.(\d+)\.(\d+)'
     family_replacement: 'Smart Lenovo Browser'
 
   # Atom Browser
@@ -678,6 +723,46 @@ user_agent_parsers:
   - regex: '(Wolvic)/(\d+)\.(\d+)\.(\d+)'
     family_replacement: 'Wolvic Browser'
 
+  # SmartTV WebBrowser
+  - regex: '(Thano)/(\d+)\.(\d+)'
+    family_replacement: 'SmartTV WebBrowser'
+
+  # WeChat Browser
+  - regex: '(MicroMessenger)/(\d+)\.(\d+)(?:\.(\d+)|)'
+    family_replacement: 'WeChat Browser'
+
+  # Odin Browser
+  - regex: '(Odin)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Odin'
+  
+  # NetCast Smart TV
+  - regex: '(Colt)/(\d+)\.(\d+)'
+    family_replacement: 'NetCast Smart TV'
+  # Lite Browser
+  - regex: '(Lite Browser)/(\d+)\.(\d+)'
+    family_replacement: 'Lite Browser'
+
+  # Vewd Browser
+  - regex: '(OMI)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Vewd Browser'
+
+  # Mypal
+  - regex: '(Mypal)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Mypal Browser'
+
+  # Chess.com native app
+  - regex: '(Chesscom-Android)/(\d+)\.(\d+)\.(\d+)'
+
+  # Roblox native app
+  - regex: '(RobloxApp)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Roblox App'
+
+  # Roadrunner iOS app (not the legacy Time Warner Cable ISP identifier)
+  - regex: '(Roadrunner)/IOS/\d+/(\d+)\.(\d+)\.(\d+)'
+
+  # Ancestry.com Android app
+  - regex: '(AncestryAndroid)/(\d+)\.(\d+)(?:\.(\d+)|)'
+
   #### END SPECIAL CASES TOP ####
 
   #### MAIN CASES - this catches > 50% of all browsers ####
@@ -775,6 +860,96 @@ user_agent_parsers:
   # Browser/major_version.minor_version
   - regex: '(bingbot|Bolt|AdobeAIR|Jasmine|IceCat|Skyfire|Midori|Maxthon|Lynx|Arora|IBrowse|Dillo|Camino|Shiira|Fennec|Phoenix|Flock|Netscape|Lunascape|Epiphany|WebPilot|Opera Mini|Opera|NetFront|Netfront|Konqueror|Googlebot|SeaMonkey|Kazehakase|Vienna|Iceape|Iceweasel|IceWeasel|Iron|K-Meleon|Sleipnir|Galeon|GranParadiso|iCab|iTunes|MacAppStore|NetNewsWire|Space Bison|Stainless|Orca|Dolfin|BOLT|Minimo|Tizen Browser|Polaris|Abrowser|Planetweb|ICE Browser|mDolphin|qutebrowser|Otter|QupZilla|MailBar|kmail2|YahooMobileMail|ExchangeWebServices|ExchangeServicesClient|Dragon|Outlook-iOS-Android)/(\d+)\.(\d+)(?:\.(\d+)|)'
 
+  # Qt Web Engine embedded browser, must be before Chrome
+  - regex: '(QtWebEngine)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Qt Web Engine'
+
+  # OpenWave browser (Chromium-based), must be before Chrome
+  - regex: '(OpenWave)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Open Wave'
+
+  # AtContent - confirmed APT29/Nobelium (Cozy Bear) C2 malware marker. The implant
+  # (AcroSup.dll, side-loaded via Adobe WCChromeNativeMessagingHost.exe) uses a hardcoded
+  # UA of the form 'Chrome/100.0.4896.75 Safari/537.36 AtContent/91.5.2444.45' to
+  # communicate with Dropbox C2. Also observed appended after Edg/ tokens.
+  # Source: Cluster25/DuskRise 'Cozy Smuggled Into the Box', May 2022
+  # (https://www.duskrise.com/2022/05/13/cozy-smuggled-into-the-box-apt29-abusing-legitimate-software-for-targeted-operations-in-europe/)
+
+  - regex: '(AtContent)/(\d+)\.(\d+)\.(\d+)'
+  # Trailer - suspicious fake UA token appended to Chrome/Edge/Opera UA strings
+  # (TOKEN/MAJOR.MINOR.BUILD.PATCH). No known legitimate browser uses this token.
+  # Structurally identical to AtContent (confirmed APT29/Nobelium C2 marker; see
+  # Cluster25/DuskRise 'Cozy Smuggled Into the Box', May 2022). Unconfirmed attribution;
+  # may be same actor rotating token names or a copycat using the same spoofing technique.
+  - regex: '(Trailer)/(\d+)\.(\d+)\.(\d+)'
+
+  # Agency - suspicious fake UA token appended to Chrome UA strings
+  # (TOKEN/MAJOR.MINOR.BUILD.PATCH). No known legitimate browser uses this token.
+  # Structurally identical to AtContent (confirmed APT29/Nobelium C2 marker; see
+  # Cluster25/DuskRise 'Cozy Smuggled Into the Box', May 2022). Unconfirmed attribution;
+  # may be same actor rotating token names or a copycat using the same spoofing technique.
+  - regex: '(Agency)/(\d+)\.(\d+)\.(\d+)'
+
+  # Herring - suspicious fake UA token appended to Chrome UA strings
+  # (TOKEN/MAJOR.MINOR.BUILD.PATCH). No known legitimate browser uses this token.
+  # Structurally identical to AtContent (confirmed APT29/Nobelium C2 marker; see
+  # Cluster25/DuskRise 'Cozy Smuggled Into the Box', May 2022). Unconfirmed attribution;
+  # may be same actor rotating token names or a copycat using the same spoofing technique.
+  - regex: '(Herring)/(\d+)\.(\d+)\.(\d+)'
+
+  # Config - suspicious fake UA token appended to Chrome UA strings
+  # (TOKEN/MAJOR.MINOR.BUILD.PATCH). No known legitimate browser uses this token.
+  # Structurally identical to AtContent (confirmed APT29/Nobelium C2 marker; see
+  # Cluster25/DuskRise 'Cozy Smuggled Into the Box', May 2022). Unconfirmed attribution;
+  # may be same actor rotating token names or a copycat using the same spoofing technique.
+  - regex: '(Config)/(\d+)\.(\d+)\.(\d+)'
+
+  # Viewer - suspicious fake UA token appended to Chrome UA strings
+  # (TOKEN/MAJOR.MINOR.BUILD.PATCH). No known legitimate browser uses this token.
+  # Structurally identical to AtContent (confirmed APT29/Nobelium C2 marker; see
+  # Cluster25/DuskRise 'Cozy Smuggled Into the Box', May 2022). Unconfirmed attribution;
+  # may be same actor rotating token names or a copycat using the same spoofing technique.
+  - regex: '(Viewer)/(\d+)\.(\d+)\.(\d+)'
+
+  # LikeWise - suspicious fake UA token appended to Chrome UA strings
+  # (TOKEN/MAJOR.MINOR.BUILD.PATCH). No known legitimate browser uses this token.
+  # Structurally identical to AtContent (confirmed APT29/Nobelium C2 marker; see
+  # Cluster25/DuskRise 'Cozy Smuggled Into the Box', May 2022). Unconfirmed attribution;
+  # may be same actor rotating token names or a copycat using the same spoofing technique.
+  - regex: '(LikeWise)/(\d+)\.(\d+)\.(\d+)'
+
+  # Unique - suspicious fake UA token appended to Chrome/Opera UA strings
+  # (TOKEN/MAJOR.MINOR.BUILD.PATCH). No known legitimate browser uses this token.
+  # Structurally identical to AtContent (confirmed APT29/Nobelium C2 marker; see
+  # Cluster25/DuskRise 'Cozy Smuggled Into the Box', May 2022). Unconfirmed attribution;
+  # may be same actor rotating token names or a copycat using the same spoofing technique.
+  - regex: '(Unique)/(\d+)\.(\d+)\.(\d+)'
+
+  # CitizenFX - embedded Chromium browser in FiveM/RedM (GTA V / RDR2 game mod frameworks)
+  - regex: '(CitizenFX)/(\d+)\.(\d+)\.(\d+)'
+
+  # R2Client - R2Games game launcher embedded browser (CEF-based)
+  - regex: '(R2Client)/(\d+)\.(\d+)(?:\.(\d+)|)'
+
+  # OBS Studio embedded browser (CEF-based, used for browser sources/docks)
+  - regex: '(OBS)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'OBS Studio'
+
+  # Adobe CEP - embedded Chromium runtime for extension panels in Adobe CC apps
+  - regex: '(AdobeCEP)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Adobe CEP'
+
+  # Steam embedded browsers; version from Chrome. Must be before Chrome.
+  # GameOverlay = in-game overlay browser (Shift+Tab)
+  - regex: 'Valve Steam (GameOverlay).{1,200}Chrome/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Steam GameOverlay'
+  # Steam Deck built-in browser
+  - regex: 'Valve Steam (Gamepad)/Steam Deck.{1,200}Chrome/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Steam Deck'
+  # Steam desktop client browser
+  - regex: '(Valve(?: Steam|) Client).{1,200}Chrome/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Steam Client'
+
   # Chrome/Chromium/major_version.minor_version
   - regex: '(Chromium|Chrome)/(\d+)\.(\d+)(?:\.(\d+)|)(?:\.(\d+)|)'
 

docs/openapi.yaml

@@ -13,7 +13,7 @@ info:
     Multiple users are supported via `X-Api-Key` headers. Create, list and delete keys via the [CLI](https://github.com/sjdonado/bit/blob/master/SETUP.md#cli).
   version: 1.6.0
   contact:
-    name: Juan Rodriguez Donado
+    name: sjdonado
     url: https://sjdonado.com
 
 servers: