chore: upgrade version to 1.5.1

chore: update API documentation
test: update cursor-based pagination test cases
2025-03-16 18:56:33 +01:00 · 2025-03-16 18:55:18 +01:00 · 2025-03-16 18:55:02 +01:00 · 2025-03-16 18:34:46 +01:00 · 2025-03-16 18:30:52 +01:00 · 2025-03-16 18:04:36 +01:00
24 changed files with 742 additions and 169 deletions
@@ -31,11 +31,14 @@ jobs:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

-      - name: Extract version
+      - name: Determine version
        id: version
        run: |
-          VERSION=$(grep '^version:' shard.yml | cut -d ' ' -f 2)
-          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          if [ "${{ github.event_name }}" = "release" ]; then
+            echo "version=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT
+          else
+            echo "version=latest" >> $GITHUB_OUTPUT
+          fi

      - name: Build and push platform image
        uses: docker/build-push-action@v5
@@ -46,7 +49,7 @@ jobs:
          platforms: ${{ matrix.platform }}
          push: true
          tags: |
-            sjdonado/bit:${{ github.event_name == 'release' && steps.version.outputs.version || 'latest' }}-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
+            sjdonado/bit:${{ steps.version.outputs.version }}-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
          build-args: |
            TARGETARCH=${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
          cache-from: type=gha
@@ -65,8 +68,17 @@ jobs:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

+      - name: Determine version
+        id: version
+        run: |
+          if [ "${{ github.event_name }}" = "release" ]; then
+            echo "version=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT
+          else
+            echo "version=latest" >> $GITHUB_OUTPUT
+          fi
+
      - name: Create manifest
        run: |
          docker buildx imagetools create \
-            -t sjdonado/bit:${{ github.event_name == 'release' && needs.build-platforms.outputs.version || 'latest' }} \
-            sjdonado/bit:${{ github.event_name == 'release' && needs.build-platforms.outputs.version || 'latest' }}-{amd64,arm64}
+            -t sjdonado/bit:${{ steps.version.outputs.version }} \
+            sjdonado/bit:${{ steps.version.outputs.version }}-{amd64,arm64}
@@ -2,18 +2,18 @@
 [![Docker Stars](https://img.shields.io/docker/stars/sjdonado/bit.svg)](https://hub.docker.com/r/sjdonado/bit)
 [![Docker Image Size](https://img.shields.io/docker/image-size/sjdonado/bit/latest)](https://hub.docker.com/r/sjdonado/bit)

-# Bit URL Shortener
-
 Lightweight URL shortener service with minimal resource requirements. Average memory consumption is **20MB RAM** with container disk space under **50MB**.

-Bit is highly performant, achieving over 850 requests per second with an average latency of just 118ms. For detailed benchmark results, see [benchmark](docs/SETUP.md#benchmark).
+Bit is highly performant, achieving over 1.8K requests per second with an average latency of 68ms. For detailed benchmark results, see [benchmark](docs/SETUP.md#benchmark).

 Images available on [Docker Hub](https://hub.docker.com/r/sjdonado/bit/tags).

-## Quick Start
-```bash
-docker run -p 4000:4000 -e ADMIN_API_KEY=$(openssl rand -base64 32) sjdonado/bit:latest
-```
+## Why Bit?
+It is feature-complete by design. Its strength lies in simplicity, a reliable URL shortener without unnecessary bloat. Bug fixes will continue, but new features aren't planned.
+
+- Minimal tracking setup: Country, browser, os, referer. No cookies or persistent tracking mechanisms are used beyond what's available from a basic client's request.
+- Flexible request forwarding system passes client context (IP, user-agent) to destinations via standard `X-Forwarded-For` and `User-Agent` headers, enabling advanced tracking and integration capabilities when needed.
+- Multiple users are supported via API key authentication. Create, list and delete via the [CLI](docs/SETUP.md#cli).

 ## Minimum Requirements
 - 50MB disk space
@@ -1,9 +1,11 @@
 require "uuid"
 require "user_agent_parser"

-UserAgent.load_regexes(File.read("data/regexes.yaml"))
-
 require "../lib/controller.cr"
+require "../lib/ip_lookup"
+
+UserAgent.load_regexes(File.read("data/uap_core_regexes.yaml"))
+IpLookup.load_mmdb("data/GeoLite2-Country.mmdb")

 module App::Controllers::Link
  class Create < App::Lib::BaseController
@@ -51,34 +53,40 @@ module App::Controllers::Link
      link = Database.get_by(Link, slug: slug)
      raise App::NotFoundException.new(env) if !link

+      remote_address = env.request.headers["Cf-Connecting-Ip"]?.try(&.presence) || env.request.remote_address.try &.to_s
+      user_agent_str = env.request.headers["User-Agent"]? || "Unknown"
+
+      client_ip = IpLookup.extract_ip(remote_address) || "Unknown"
+
+      env.response.status_code = 301
+      env.response.headers["Location"] = link.url!
+
+      env.response.headers["X-Forwarded-For"] = client_ip
+      env.response.headers["User-Agent"] = user_agent_str
+
      spawn do
-        user_agent_str = env.request.headers["User-Agent"]? || "Unknown"
+        ip_lookup = client_ip != "Unknown" ? IpLookup.new(client_ip) : nil
+        country = ip_lookup.try &.country.try &.code
+
        user_agent = user_agent_str != "Unknown" ? UserAgent.new(user_agent_str) : nil

-        language_header = env.request.headers["Accept-Language"]? || "Unknown"
-        language = language_header.split(',').first.split(';').first
-
-        referer = env.request.headers["Referer"]?
+        source = env.params.query["utm_source"]? || "Direct"
+        referer_host = env.request.headers["Referer"]?.try { |r| begin URI.parse(r).host rescue r end } || source

        click = Click.new
        click.id = UUID.v4.to_s
        click.link = link
-        click.language = language
+        click.country = country
        click.user_agent = user_agent_str
-        click.browser = user_agent ? user_agent.family : "Unknown"
-        click.os = user_agent ? (user_agent.os.try &.family || "Unknown") : "Unknown"
-        click.source = referer ? URI.parse(referer).host : "Unknown"
+        click.browser = user_agent.try &.family
+        click.os = user_agent.try &.os.try &.family
+        click.referer = referer_host

        changeset = Database.insert(click)
        if changeset.errors.any?
          Log.error { "Logging click event failed: #{changeset.errors}" }
        end
      end
-
-      env.response.status_code = 301
-      env.response.headers["Location"] = link.url!
-      env.response.headers["Content-Type"] = "text/html"
-      env.response.print("Redirecting...")
    end
  end

@@ -89,10 +97,30 @@ module App::Controllers::Link
    def call(env)
      user = env.get("user").as(User)

-      query = Database::Query.where(user_id: user.id.as(String))
-      links = Database.all(Link, query, preload: [:clicks])
+      limit = (env.params.query["limit"]? || "100").to_i
+      cursor = env.params.query["cursor"]?
+
+      query = Database::Query.where(user_id: user.id.as(String))
+      if cursor
+        query = query.where("id < ?", cursor)
+      end
+
+      query = query.order_by("id DESC").limit(limit + 1)
+      links = Database.all(Link, query)
+
+      has_more = links.size > limit
+      links = links[0...limit] if has_more
+
+      next_cursor = has_more ? links.last.id : nil
+
+      response = {
+        "data" => links.map { |link| App::Serializers::Link.new(link) },
+        "pagination" => {
+          "has_more" => has_more,
+          "next" => next_cursor
+        }
+      }

-      response = {"data" => links.map { |link| App::Serializers::Link.new(link) }}
      response.to_json
    end
  end
@@ -106,15 +134,58 @@ module App::Controllers::Link
      link_id = env.params.url["id"]

      query = Database::Query.where(id: link_id.as(String), user_id: user.id.as(String)).limit(1)
-      link = Database.all(Link, query, preload: [:clicks]).first?
-
+      link = Database.all(Link, query).first?
      raise App::NotFoundException.new(env) if link.nil?

+      clicks_query = Database::Query.where(link_id: link_id.as(String))
+                                    .order_by("id DESC")
+                                    .limit(100)
+      link.clicks = Database.all(Click, clicks_query)
+
      response = {"data" => App::Serializers::Link.new(link)}
      response.to_json
    end
  end

+  class Clicks < App::Lib::BaseController
+    include App::Models
+    include App::Lib
+
+    def call(env)
+      user = env.get("user").as(User)
+      link_id = env.params.url["id"]
+
+      link_query = Database::Query.where(id: link_id.as(String), user_id: user.id.as(String)).limit(1)
+      link = Database.all(Link, link_query).first?
+      raise App::NotFoundException.new(env) if link.nil?
+
+      limit = (env.params.query["limit"]? || "100").to_i
+      cursor = env.params.query["cursor"]?
+
+      query = Database::Query.where(link_id: link_id.as(String))
+      if cursor
+        query = query.where("id < ?", cursor)
+      end
+
+      query = query.order_by("id DESC").limit(limit + 1)
+      clicks = Database.all(Click, query)
+
+      has_more = clicks.size > limit
+      clicks = clicks[0...limit] if has_more
+      next_cursor = has_more ? clicks.last.id : nil
+
+      response = {
+        "data" => clicks.map { |click| App::Serializers::Click.new(click) },
+        "pagination" => {
+          "has_more" => has_more,
+          "next" => next_cursor
+        }
+      }
+
+      response.to_json
+    end
+  end
+
  class Update < App::Lib::BaseController
    include App::Models
    include App::Lib
@@ -3,7 +3,7 @@ require "../lib/controller.cr"
 module App::Controllers::Ping
  class Get < App::Lib::BaseController
    def call(env)
-      response = {"pong" => "ok"}
+      response = {"data" => "pong"}
      response.to_json
    end
  end
@@ -0,0 +1,54 @@
+require "maxminddb"
+
+class IpLookup
+  @@instance : MaxMindDB::Reader? = nil
+
+  record Country, code : String? = nil, name : String? = nil
+
+  getter ip : String
+  getter country : Country?
+
+  def self.load_mmdb(mmdb_file_path : String)
+    @@instance = MaxMindDB.open(mmdb_file_path)
+  end
+
+  def initialize(ip_address : String)
+    @ip = ip_address
+    @country = nil
+
+    return if @@instance.nil? || ip_address == "Unknown" || ip_address.empty?
+
+    begin
+      lookup = @@instance.not_nil!.get(ip_address)
+
+      country_code = lookup["country"]?.try &.["iso_code"]?.try &.as_s
+      country_name = lookup["country"]?.try &.["names"]?.try &.["en"]?.try &.as_s
+
+      if country_code || country_name
+        @country = Country.new(
+          code: country_code,
+          name: country_name
+        )
+      end
+    rescue ex
+      # Silently handle lookup errors
+      Log.error { "IP lookup failed: #{ex.message}" }
+    end
+  end
+
+  def self.extract_ip(address_string : String?) : String?
+    return nil if address_string.nil?
+
+    if address_string.includes?('[') # IPv6 with port: [2001:db8::1]:8080
+      address_string.split(']').first.sub('[', '\'')
+    elsif address_string.includes?(':')
+      if address_string.count(':') > 1 # IPv6 without port
+        address_string
+      else # IPv4 with port: 192.168.1.1:8080
+        address_string.split(':').first
+      end
+    else # Address without port
+      address_string
+    end
+  end
+end
@@ -5,14 +5,14 @@ module App::Models
    schema :clicks do
      field :id, String, primary_key: true
      field :user_agent, String
-      field :language, String
+      field :country, String
      field :browser, String
      field :os, String
-      field :source, String
+      field :referer, String

      belongs_to :link, Link
    end

-    validate_required [:user_agent, :language, :source]
+    validate_required [:user_agent, :referer]
  end
 end
@@ -27,6 +27,10 @@ module App
    Controllers::Link::Get.new.call(env)
  end

+  get "/api/links/:id/clicks" do |env|
+    Controllers::Link::Clicks.new.call(env)
+  end
+
  post "/api/links" do |env|
    Controllers::Link::Create.new.call(env)
  end
@@ -11,10 +11,10 @@ module App::Serializers
      builder.object do
        builder.field("id", @click.id)
        builder.field("user_agent", @click.user_agent)
-        builder.field("language", @click.language)
+        builder.field("country", @click.country)
        builder.field("browser", @click.browser)
        builder.field("os", @click.os)
-        builder.field("source", @click.source)
+        builder.field("referer", @click.referer)
        builder.field("created_at", @click.created_at)
      end
    end
@@ -16,7 +16,15 @@ module App::Serializers
        builder.field("id", @link.id)
        builder.field("refer", @refer)
        builder.field("origin", @link.url)
-        builder.field("clicks", @link.clicks.map { |click| App::Serializers::Click.new(click) })
+
+        begin
+          clicks = @link.clicks
+          unless clicks.empty?
+            builder.field("clicks", clicks.map { |click| App::Serializers::Click.new(click) })
+          end
+        rescue Crecto::AssociationNotLoaded
+          # Association not loaded, skip this field silently
+        end
      end
    end
  end
@@ -1,3 +1,6 @@
+require "file_utils"
+require "http/client"
+
 require "../config/*"
 require "../lib/*"
 require "../models/*"
@@ -53,4 +56,86 @@ module App::Services::Cli
      puts "Admin setup skipped: Missing ADMIN_NAME or ADMIN_API_KEY environment variables."
    end
  end
+
+  def self.update_uap_regexes
+    puts "Downloading User-Agent Parser core regexes..."
+
+    FileUtils.mkdir_p("data")
+    url = "https://raw.githubusercontent.com/ua-parser/uap-core/master/regexes.yaml"
+    output_file = "data/uap_core_regexes.yaml"
+
+    begin
+      http_get_with_redirect(url) do |response|
+        File.write(output_file, response.body_io.gets_to_end)
+      end
+      puts "User-Agent regexes downloaded to #{output_file}"
+    rescue e
+      puts "Error: Failed to download UAP core regexes: #{e.message}"
+    end
+  end
+
+  def self.download_geolite_db
+    puts "Downloading GeoLite2 Country database..."
+
+    FileUtils.mkdir_p("data")
+    url = "https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-Country.mmdb"
+    output_file = "data/GeoLite2-Country.mmdb"
+
+    begin
+      File.open(output_file, "wb") do |file|
+        http_get_with_redirect(url) do |response|
+          IO.copy(response.body_io, file)
+        end
+      end
+      puts "GeoLite2 database downloaded to #{output_file}"
+    rescue e
+      puts "Error: Failed to download GeoLite2 database: #{e.message}"
+    end
+  end
+
+  private def self.http_get_with_redirect(url : String, max_redirects = 5)
+    redirects = 0
+
+    while redirects < max_redirects
+      uri = URI.parse(url)
+      client = HTTP::Client.new(uri)
+
+      success = false
+      follow_redirect = false
+      redirect_url = nil
+
+      begin
+        client.get(uri.request_target) do |response|
+          case response.status_code
+          when 200
+            yield response
+            success = true
+          when 301, 302
+            if new_location = response.headers["Location"]?
+              puts "Following redirect to: #{new_location}"
+              redirect_url = new_location
+              follow_redirect = true
+            else
+              raise "Received redirect status but no Location header"
+            end
+          else
+            raise "Failed request with status code: #{response.status_code}"
+          end
+        end
+      ensure
+        client.close
+      end
+
+      return if success
+
+      if follow_redirect && redirect_url
+        url = redirect_url
+        redirects += 1
+      else
+        break
+      end
+    end
+
+    raise "Too many redirects (#{max_redirects})"
+  end
 end
@@ -1,3 +0,0 @@
-module App
-  VERSION = "0.1.0"
-end
@@ -148,11 +148,11 @@ user_agent_parsers:
    family_replacement: 'Pinterestbot'

  # Bots
-  - regex: '(CSimpleSpider|Cityreview Robot|CrawlDaddy|CrawlFire|Finderbots|Index crawler|Job Roboter|KiwiStatus Spider|Lijit Crawler|QuerySeekerSpider|ScollSpider|Trends Crawler|USyd-NLP-Spider|SiteCat Webbot|BotName\/\$BotVersion|123metaspider-Bot|1470\.net crawler|50\.nu|8bo Crawler Bot|Aboundex|Accoona-[A-z]{1,30}-Agent|AdsBot-Google(?:-[a-z]{1,30}|)|altavista|AppEngine-Google|archive.{0,30}\.org_bot|archiver|Ask Jeeves|[Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]{1,30})(?:-[A-Za-z]{1,30}|)|bingbot|BingPreview|blitzbot|BlogBridge|Bloglovin|BoardReader Blog Indexer|BoardReader Favicon Fetcher|boitho.com-dc|BotSeer|BUbiNG|\b\w{0,30}favicon\w{0,30}\b|\bYeti(?:-[a-z]{1,30}|)|Catchpoint(?: bot|)|[Cc]harlotte|Checklinks|clumboot|Comodo HTTP\(S\) Crawler|Comodo-Webinspector-Crawler|ConveraCrawler|CRAWL-E|CrawlConvera|Daumoa(?:-feedfetcher|)|Feed Seeker Bot|Feedbin|findlinks|Flamingo_SearchEngine|FollowSite Bot|furlbot|Genieo|gigabot|GomezAgent|gonzo1|(?:[a-zA-Z]{1,30}-|)Googlebot(?:-[a-zA-Z]{1,30}|)|Google SketchUp|grub-client|gsa-crawler|heritrix|HiddenMarket|holmes|HooWWWer|htdig|ia_archiver|ICC-Crawler|Icarus6j|ichiro(?:/mobile|)|IconSurf|IlTrovatore(?:-Setaccio|)|InfuzApp|Innovazion Crawler|InternetArchive|IP2[a-z]{1,30}Bot|jbot\b|KaloogaBot|Kraken|Kurzor|larbin|LEIA|LesnikBot|Linguee Bot|LinkAider|LinkedInBot|Lite Bot|Llaut|lycos|Mail\.RU_Bot|masscan|masidani_bot|Mediapartners-Google|Microsoft .{0,30} Bot|mogimogi|mozDex|MJ12bot|msnbot(?:-media {0,2}|)|msrbot|Mtps Feed Aggregation System|netresearch|Netvibes|NewsGator[^/]{0,30}|^NING|Nutch[^/]{0,30}|Nymesis|ObjectsSearch|OgScrper|Orbiter|OOZBOT|PagePeeker|PagesInventory|PaxleFramework|Peeplo Screenshot Bot|PHPCrawl|PlantyNet_WebRobot|Pompos|Qwantify|Read%20Later|Reaper|RedCarpet|Retreiver|Riddler|Rival IQ|scooter|Scrapy|Scrubby|searchsight|seekbot|semanticdiscovery|SemrushBot|Simpy|SimplePie|SEOstats|SimpleRSS|SiteCon|Slackbot-LinkExpanding|Slack-ImgProxy|Slurp|snappy|Speedy Spider|Squrl Java|Stringer|TheUsefulbot|ThumbShotsBot|Thumbshots\.ru|Tiny Tiny RSS|Twitterbot|WhatsApp|URL2PNG|Vagabondo|VoilaBot|^vortex|Votay bot|^voyager|WASALive.Bot|Web-sniffer|WebThumb|WeSEE:[A-z]{1,30}|WhatWeb|WIRE|WordPress|Wotbox|www\.almaden\.ibm\.com|Xenu(?:.s|) Link Sleuth|Xerka [A-z]{1,30}Bot|yacy(?:bot|)|YahooSeeker|Yahoo! Slurp|Yandex\w{1,30}|YodaoBot(?:-[A-z]{1,30}|)|YottaaMonitor|Yowedo|^Zao|^Zao-Crawler|ZeBot_www\.ze\.bz|ZooShot|ZyBorg|ArcGIS Hub Indexer)(?:[ /]v?(\d+)(?:\.(\d+)(?:\.(\d+)|)|)|)'
+  - regex: '(CSimpleSpider|Cityreview Robot|CrawlDaddy|CrawlFire|Finderbots|Index crawler|Job Roboter|KiwiStatus Spider|Lijit Crawler|QuerySeekerSpider|ScollSpider|Trends Crawler|USyd-NLP-Spider|SiteCat Webbot|BotName\/\$BotVersion|123metaspider-Bot|1470\.net crawler|50\.nu|8bo Crawler Bot|Aboundex|Accoona-[A-z]{1,30}-Agent|AdsBot-Google(?:-[a-z]{1,30}|)|altavista|AppEngine-Google|archive.{0,30}\.org_bot|archiver|Ask Jeeves|[Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]{1,30})(?:-[A-Za-z]{1,30}|)|bingbot|BingPreview|blitzbot|BlogBridge|Bloglovin|BoardReader Blog Indexer|BoardReader Favicon Fetcher|boitho.com-dc|BotSeer|BUbiNG|\b\w{0,30}favicon\w{0,30}\b|\bYeti(?:-[a-z]{1,30}|)|Catchpoint(?: bot|)|[Cc]harlotte|Checklinks|clumboot|Comodo HTTP\(S\) Crawler|Comodo-Webinspector-Crawler|ConveraCrawler|CRAWL-E|CrawlConvera|Daumoa(?:-feedfetcher|)|Feed Seeker Bot|Feedbin|findlinks|Flamingo_SearchEngine|FollowSite Bot|furlbot|Genieo|gigabot|GomezAgent|gonzo1|(?:[a-zA-Z]{1,30}-|)Googlebot(?:-[a-zA-Z]{1,30}|)|GoogleOther|Google SketchUp|grub-client|gsa-crawler|heritrix|HiddenMarket|holmes|HooWWWer|htdig|ia_archiver|ICC-Crawler|Icarus6j|ichiro(?:/mobile|)|IconSurf|IlTrovatore(?:-Setaccio|)|InfuzApp|Innovazion Crawler|InternetArchive|IP2[a-z]{1,30}Bot|jbot\b|KaloogaBot|Kraken|Kurzor|larbin|LEIA|LesnikBot|Linguee Bot|LinkAider|LinkedInBot|Lite Bot|Llaut|lycos|Mail\.RU_Bot|masscan|masidani_bot|Mediapartners-Google|Microsoft .{0,30} Bot|mogimogi|mozDex|MJ12bot|msnbot(?:-media {0,2}|)|msrbot|Mtps Feed Aggregation System|netresearch|Netvibes|NewsGator[^/]{0,30}|^NING|Nutch[^/]{0,30}|Nymesis|ObjectsSearch|OgScrper|Orbiter|OOZBOT|PagePeeker|PagesInventory|PaxleFramework|Peeplo Screenshot Bot|PHPCrawl|PlantyNet_WebRobot|Pompos|Qwantify|Read%20Later|Reaper|RedCarpet|Retreiver|Riddler|Rival IQ|scooter|Scrapy|Scrubby|searchsight|seekbot|semanticdiscovery|SemrushBot|Simpy|SimplePie|SEOstats|SimpleRSS|SiteCon|Slackbot-LinkExpanding|Slack-ImgProxy|Slurp|snappy|Speedy Spider|Squrl Java|Stringer|TheUsefulbot|ThumbShotsBot|Thumbshots\.ru|Tiny Tiny RSS|Twitterbot|WhatsApp|URL2PNG|Vagabondo|VoilaBot|^vortex|Votay bot|^voyager|WASALive.Bot|Web-sniffer|WebThumb|WeSEE:[A-z]{1,30}|WhatWeb|WIRE|WordPress|Wotbox|www\.almaden\.ibm\.com|Xenu(?:.s|) Link Sleuth|Xerka [A-z]{1,30}Bot|yacy(?:bot|)|YahooSeeker|Yahoo! Slurp|Yandex\w{1,30}|YodaoBot(?:-[A-z]{1,30}|)|YottaaMonitor|Yowedo|^Zao|^Zao-Crawler|ZeBot_www\.ze\.bz|ZooShot|ZyBorg|ArcGIS Hub Indexer|GPTBot)(?:[ /]v?(\d+)(?:\.(\d+)(?:\.(\d+)|)|)|)'

  # AWS S3 Clients
  # must come before "Bots General matcher" to catch "boto"/"boto3" before "bot"
-  - regex: '\b(Boto3?|JetS3t|aws-(?:cli|sdk-(?:cpp|go|java|nodejs|ruby2?|dotnet-(?:\d{1,2}|core)))|s3fs)/(\d+)\.(\d+)(?:\.(\d+)|)'
+  - regex: '\b(Boto3?|JetS3t|aws-(?:cli|sdk-(?:cpp|go|go-v\d|java|nodejs|ruby2?|dotnet-(?:\d{1,2}|core)))|s3fs)/(\d+)\.(\d+)(?:\.(\d+)|)'

  # SAFE FME
  - regex: '(FME)\/(\d+\.\d+)\.(\d+)\.(\d+)'
@@ -179,6 +179,9 @@ user_agent_parsers:
  - regex: '\[FB.{0,300};'
    family_replacement: 'Facebook'

+  # RecipeRadar crawler
+  - regex: '(RecipeRadar)/(\d+)\.(\d+)(?:\.(\d+)|)'
+
  # Bots General matcher 'name/0.0'
  - regex: '^.{0,200}?(?:\/[A-Za-z0-9\.]{0,50}|) {0,2}([A-Za-z0-9 \-_\!\[\]:]{0,50}(?:[Aa]rchiver|[Ii]ndexer|[Ss]craper|[Bb]ot|[Ss]pider|[Cc]rawl[a-z]{0,50}))[/ ](\d+)(?:\.(\d+)(?:\.(\d+)|)|)'
  # Bots containing bot(but not CUBOT)
@@ -215,6 +218,16 @@ user_agent_parsers:
  # Twitter
  - regex: '(Twitter for (?:iPhone|iPad)|TwitterAndroid)(?:\/(\d+)\.(\d+)|)'
    family_replacement: 'Twitter'
+  # TikTok
+  - regex: '(musical_ly) app_version\/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'TikTok'
+  - regex: '(musical_ly_)(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'TikTok'
+  - regex: '(BytedanceWebview)\/[a-z0-9]+'
+    family_replacement: 'TikTok'
+  # KakaoTalk
+  - regex: 'Mozilla.{1,200}Mobile.{1,100}(KAKAOTALK)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'KakaoTalk'

  # Phantom app
  - regex: 'Mozilla.{1,200}Mobile.{1,100}(Phantom\/ios|Phantom\/android).(\d+)\.(\d+)\.(\d+)'
@@ -394,7 +407,7 @@ user_agent_parsers:
  - regex: '(Instabridge)/(\d+)(?:\.(\d+)|)(?:\.(\d+)|)'

  # Aloha Browser
-  - regex: '(AlohaBrowser)/(\d+)\.(\d+)\.(\d+)(?:\.(\d+)|)'
+  - regex: '(AlohaBrowser|ABB)/(\d+)\.(\d+)\.(\d+)(?:\.(\d+)|)'
    family_replacement: 'Aloha Browser'

  # Brave Browser https://brave.com/ , should go before Safari and Chrome Mobile
@@ -415,7 +428,7 @@ user_agent_parsers:
    family_replacement: 'Edge Mobile'

  # Oculus Browser, should go before Samsung Internet
-  - regex: '(OculusBrowser)/(\d+)\.(\d+).0.0(?:\.([0-9\-]+)|)'
+  - regex: '(OculusBrowser)/(\d+)\.(\d+)(?:\.([0-9\-]+)|)'
    family_replacement: 'Oculus Browser'

  # Samsung Internet (based on Chrome, but lacking some features)
@@ -492,6 +505,12 @@ user_agent_parsers:
  - regex: '(Ecosia) android@(\d+)(?:\.(\d+)|)(?:\.(\d+)|)(?:\.(\d+)|)'
    family_replacement: 'Ecosia Android'

+  # VivoBrowser
+  - regex: '(VivoBrowser)\/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+
+  # HiBrowser
+  - regex: '(HiBrowser)\/v(\d+)\.(\d+)\.(\d+)\.(\d+)'
+
  # Chrome Mobile
  - regex: 'Version/.{1,300}(Chrome)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
    family_replacement: 'Chrome Mobile WebView'
@@ -569,6 +588,85 @@ user_agent_parsers:
  - regex: '^(surveyon)/(\d+)\.(\d+)\.(\d+)'
    family_replacement: 'Surveyon'

+  # 115 Browser
+  - regex: '(115Browser)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: '115 Browser'
+
+ # Avira
+  - regex: '(Avira)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Avira'
+
+  # CCleaner Browser
+  - regex: '(CCleaner)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'CCleaner'
+
+  # Norton
+  - regex: '(Norton)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Norton'
+
+  # Quark
+  - regex: '(Quark)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Quark'
+  # Quark PC
+  - regex: '(QuarkPC)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Quark PC'
+
+  # Smart Lenovo Browser
+  - regex: '(SLBrowser)/(\d+)\.(\d+)\.(\d+)\.(\d+) SLBChan/(\d+)'
+    family_replacement: 'Smart Lenovo Browser'
+  
+  # Atom Browser
+  - regex: '(Atom)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Atom Browser'
+
+  # 360 Secure Browser
+  - regex: '(Chrome)/\d+\.\d+\.\d+\.\d+ .* QIHU 360(?:SEi18n|ENT)'
+    family_replacement: '360 Secure Browser'
+
+  # Decentr Web3 Browser
+  - regex: '(Decentr)'
+    family_replacement: 'Decentr Web3 Browser'
+
+  # Sparrow Browser
+  - regex: '(Sparrow)'
+    family_replacement: 'Sparrow Browser'
+
+  # Chromium GOST Browser
+  - regex: '(Chromium GOST)'
+    family_replacement: 'Chromium GOST Browser'
+
+  # AOL Shield Browser
+  - regex: '(AOLShield)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'AOL Shield Browser'
+
+  # Hola Browser
+  - regex: '(Hola)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Hola Browser'
+
+  # Craving Explorer Browser
+  - regex: '(CravingExplorer)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Craving Explorer Browser'
+
+  # Talon Cyber Security Browser
+  - regex: '(Talon)'
+    family_replacement: 'Talon Cyber Security Browser'
+
+  # QAX Browser
+  - regex: '(Qaxbrowser)'
+    family_replacement: 'QAX Browser'
+
+  # AOL Desktop Gold Browser
+  - regex: '(ADG)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'AOL Desktop Gold Browser'
+
+  # Sber Browser
+  - regex: '(SberBrowser)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Sber Browser'
+
+  # JiSu Browser
+  - regex: '(JiSu)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'JiSu Browser'
+
  #### END SPECIAL CASES TOP ####

  #### MAIN CASES - this catches > 50% of all browsers ####
@@ -1387,6 +1485,13 @@ os_parsers:
  # Box Drive and Box Sync on Mac OS X use OSX version numbers, not Darwin
  - regex: '^Box.{0,200};(Darwin)/(10)\.(1\d)(?:\.(\d+)|)'
    os_replacement: 'Mac OS X'
+  
+  ##########
+  # Hashicorp API
+  # APN/1.0 HashiCorp/1.0 Terraform/1.8.0 (+https://www.terraform.io) terraform-provider-aws/4.67.0 (+https://registry.terraform.io/providers/hashicorp/aws) aws-sdk-go/1.44.261 (go1.19.8; darwin; arm64)
+  ##########
+  - regex: 'darwin; arm64'
+    os_replacement: 'Mac OS X'

  ##########
  # iOS
@@ -1672,29 +1777,27 @@ os_parsers:
  - regex: 'CFNetwork/.{0,100} Darwin/(21)\.\d+'
    os_replacement: 'iOS'
    os_v1_replacement: '15'
-  - regex: 'CFNetwork/.{0,100} Darwin/22\.0\.\d+'
+  - regex: 'CFNetwork/.{0,100} Darwin/22\.([0-5])\.\d+'
    os_replacement: 'iOS'
    os_v1_replacement: '16'
-    os_v2_replacement: '0'
-  - regex: 'CFNetwork/.{0,100} Darwin/22\.1\.\d+'
-    os_replacement: 'iOS'
-    os_v1_replacement: '16'
-    os_v2_replacement: '1'
-  - regex: 'CFNetwork/.{0,100} Darwin/22\.2\.\d+'
-    os_replacement: 'iOS'
-    os_v1_replacement: '16'
-    os_v2_replacement: '2'
-  - regex: 'CFNetwork/.{0,100} Darwin/22\.3\.\d+'
-    os_replacement: 'iOS'
-    os_v1_replacement: '16'
-    os_v2_replacement: '3'
-  - regex: 'CFNetwork/.{0,100} Darwin/22\.4\.\d+'
-    os_replacement: 'iOS'
-    os_v1_replacement: '16'
-    os_v2_replacement: '4'
+    os_v2_replacement: '$1'
  - regex: 'CFNetwork/.{0,100} Darwin/(22)\.\d+'
    os_replacement: 'iOS'
    os_v1_replacement: '16'
+  - regex: 'CFNetwork/.{0,100} Darwin/23\.([0-5])\.\d+'
+    os_replacement: 'iOS'
+    os_v1_replacement: '17'
+    os_v2_replacement: '$1'
+  - regex: 'CFNetwork/.{0,100} Darwin/(23)\.\d+'
+    os_replacement: 'iOS'
+    os_v1_replacement: '17'
+  - regex: 'CFNetwork/.{0,100} Darwin/24\.([0-5])\.\d+'
+    os_replacement: 'iOS'
+    os_v1_replacement: '18'
+    os_v2_replacement: '$1'
+  - regex: 'CFNetwork/.{0,100} Darwin/(24)\.\d+'
+    os_replacement: 'iOS'
+    os_v1_replacement: '18'
  - regex: 'CFNetwork/.{0,100} Darwin/'
    os_replacement: 'iOS'

@@ -1889,6 +1992,21 @@ os_parsers:
  # Roku Digital-Video-Players https://www.roku.com/
  - regex: '^(Roku)/DVP-(\d+)\.(\d+)'

+  ##########
+  # Amazon S3 client boto3
+  # Hasicorp API
+  # Boto3/1.28.62 md/Botocore#1.31.62 ua/2.0 os/macos#22.4.0 md/arch#arm64 lang/python#3.11.6 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.62
+  # APN/1.0 HashiCorp/1.0 Terraform/1.8.1 (+https://www.terraform.io) terraform-provider-aws/4.67.0 (+https://registry.terraform.io/providers/hashicorp/aws) aws-sdk-go-v2/1.18.0 os/macos lang/go/1.19.8 md/GOOS/darwin md/GOARCH/arm64 api/identitystore/1.16.11
+  ##########
+  - regex: 'os\/macos[#]?(\d*)[.]?(\d*)[.]?(\d*)'
+    os_replacement: 'Mac OS X'
+    os_v1_replacement: '$1'
+    os_v2_replacement: '$2'
+    os_v3_replacement: '$3'
+
+  # Huawei HarmonyOS
+  - regex: '(HarmonyOS)[\s;]+(\d+|)\.?(\d+|)\.?(\d+|)'
+
 device_parsers:

  #########
@@ -5905,7 +6023,7 @@ device_parsers:
  ##########
  # Spiders (this is a hack...)
  ##########
-  - regex: '^.{0,100}(bot|BUbiNG|zao|borg|DBot|oegp|silk|Xenu|zeal|^NING|CCBot|crawl|htdig|lycos|slurp|teoma|voila|yahoo|Sogou|CiBra|Nutch|^Java/|^JNLP/|Daumoa|Daum|Genieo|ichiro|larbin|pompos|Scrapy|snappy|speedy|spider|msnbot|msrbot|vortex|^vortex|crawler|favicon|indexer|Riddler|scooter|scraper|scrubby|WhatWeb|WinHTTP|bingbot|BingPreview|openbot|gigabot|furlbot|polybot|seekbot|^voyager|archiver|Icarus6j|mogimogi|Netvibes|blitzbot|altavista|charlotte|findlinks|Retreiver|TLSProber|WordPress|SeznamBot|ProoXiBot|wsr\-agent|Squrl Java|EtaoSpider|PaperLiBot|SputnikBot|A6\-Indexer|netresearch|searchsight|baiduspider|YisouSpider|ICC\-Crawler|http%20client|Python-urllib|dataparksearch|converacrawler|Screaming Frog|AppEngine-Google|YahooCacheSystem|fast\-webcrawler|Sogou Pic Spider|semanticdiscovery|Innovazion Crawler|facebookexternalhit|Google.{0,200}/\+/web/snippet|Google-HTTP-Java-Client|BlogBridge|IlTrovatore-Setaccio|InternetArchive|GomezAgent|WebThumbnail|heritrix|NewsGator|PagePeeker|Reaper|ZooShot|holmes|NL-Crawler|Pingdom|StatusCake|WhatsApp|masscan|Google Web Preview|Qwantify|Yeti|OgScrper)'
+  - regex: '^.{0,100}(bot|BUbiNG|zao|borg|DBot|oegp|silk|Xenu|zeal|^NING|CCBot|crawl|htdig|lycos|slurp|teoma|voila|yahoo|Sogou|CiBra|Nutch|^Java/|^JNLP/|Daumoa|Daum|Genieo|ichiro|larbin|pompos|Scrapy|snappy|speedy|spider|msnbot|msrbot|vortex|^vortex|crawler|favicon|indexer|Riddler|scooter|scraper|scrubby|WhatWeb|WinHTTP|bingbot|BingPreview|openbot|gigabot|furlbot|polybot|seekbot|^voyager|archiver|Icarus6j|mogimogi|Netvibes|blitzbot|altavista|charlotte|findlinks|Retreiver|TLSProber|WordPress|SeznamBot|ProoXiBot|wsr\-agent|Squrl Java|EtaoSpider|PaperLiBot|SputnikBot|A6\-Indexer|netresearch|searchsight|baiduspider|YisouSpider|ICC\-Crawler|http%20client|Python-urllib|dataparksearch|converacrawler|Screaming Frog|AppEngine-Google|YahooCacheSystem|fast\-webcrawler|Sogou Pic Spider|semanticdiscovery|Innovazion Crawler|facebookexternalhit|Google.{0,200}/\+/web/snippet|Google-HTTP-Java-Client|BlogBridge|IlTrovatore-Setaccio|InternetArchive|GomezAgent|WebThumbnail|heritrix|NewsGator|PagePeeker|Reaper|ZooShot|holmes|NL-Crawler|Pingdom|StatusCake|WhatsApp|masscan|Google Web Preview|Qwantify|Yeti|OgScrper|RecipeRadar|GPTBot)'
    regex_flag: 'i'
    device_replacement: 'Spider'
    brand_replacement: 'Spider'
@@ -4,7 +4,6 @@ CREATE TABLE clicks (
 	id TEXT PRIMARY KEY NOT NULL,
 	link_id TEXT NOT NULL,
 	user_agent TEXT,
-	language TEXT,
 	browser TEXT,
 	os TEXT,
 	source TEXT,
@@ -0,0 +1,8 @@
+-- +micrate Up
+-- SQL in section 'Up' is executed when this migration is applied
+ALTER TABLE clicks ADD COLUMN country TEXT;
+ALTER TABLE clicks RENAME COLUMN source TO referer;
+
+-- +micrate Down
+-- SQL section 'Down' is executed when this migration is rolled back
+ALTER TABLE clicks RENAME COLUMN referer TO source;
@@ -0,0 +1,13 @@
+-- +micrate Up
+-- SQL in section 'Up' is executed when this migration is applied
+UPDATE clicks SET user_agent = NULL WHERE user_agent = 'Unknown';
+UPDATE clicks SET browser = NULL WHERE browser = 'Unknown';
+UPDATE clicks SET os = NULL WHERE os = 'Unknown';
+UPDATE clicks SET referer = NULL WHERE referer = 'Unknown';
+
+-- +micrate Down
+-- SQL section 'Down' is executed when this migration is rolled back
+UPDATE clicks SET user_agent = 'Unknown' WHERE user_agent IS NULL;
+UPDATE clicks SET browser = 'Unknown' WHERE browser IS NULL;
+UPDATE clicks SET os = 'Unknown' WHERE os IS NULL;
+UPDATE clicks SET referer = 'Unknown' WHERE referer IS NULL;
@@ -7,42 +7,22 @@
   - Response Example
     ```json
     {
-       "message": "pong"
+       "data": "pong"
     }
     ```
-
 2. **Redirect by Slug**

   - Endpoint: `GET /:slug`
-   - Headers: `X-Api-Key`
   - Payload: None
-   - Response Example
-     ```json
-     {
-       "data": {
-         "id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db",
-         "refer": "http://localhost:4000/3wP4BQ",
-         "origin": "https://monocuco.donado.co",
-         "clicks": [
-           {
-             "id": "730e2202-58f9-478c-a24c-f1c561df6716",
-             "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0",
-             "language": "en-US",
-             "browser": "Firefox",
-             "os": "Mac OS X",
-             "source": "Unknown",
-             "created_at": "2024-07-12T19:25:22Z"
-           }
-         ]
-       }
-     }
-     ```
+   - Response: 301

 3. **List All Links**

   - Endpoint: `GET /api/links`
   - Headers: `X-Api-Key`
-   - Payload: None
+   - Query Parameters:
+     - `limit` (optional): Number of results per page (default: 100)
+     - `cursor` (optional): Pagination cursor from previous response
   - Response Example
     ```json
     {
@@ -50,28 +30,21 @@
         {
           "id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db",
           "refer": "http://localhost:4000/3wP4BQ",
-           "origin": "https://monocuco.donado.co",
-           "clicks": [
-             {
-               "id": "730e2202-58f9-478c-a24c-f1c561df6716",
-               "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0",
-               "language": "en-US",
-               "browser": "Firefox",
-               "os": "Mac OS X",
-               "source": "Unknown",
-               "created_at": "2024-07-12T19:25:22Z"
-             }
-           ]
+           "origin": "https://monocuco.donado.co"
         }
-       ]
+       ],
+       "pagination": {
+         "has_more": true,
+         "next": "75e0a7f4-9c5e-1235-b546-eb9c5e40f7ac"
+       }
     }
     ```

 4. **List link by ID**
-
   - Endpoint: `GET /api/links/:id`
   - Headers: `X-Api-Key`
   - Payload: None
+   - Note: This endpoint returns up to 100 of the most recent clicks. For complete click history, use the `/api/links/:id/clicks` endpoint with pagination.
   - Response Example
     ```json
     {
@@ -83,10 +56,10 @@
           {
             "id": "730e2202-58f9-478c-a24c-f1c561df6716",
             "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0",
-             "language": "en-US",
+             "country": "DE",
             "browser": "Firefox",
             "os": "Mac OS X",
-             "source": "Unknown",
+             "referer": "Direct",
             "created_at": "2024-07-12T19:25:22Z"
           }
         ]
@@ -94,9 +67,35 @@
     }
     ```

-5. **Create new link**
+5. **List Clicks for a Link**
+   - Endpoint: `GET /api/links/:id/clicks`
+   - Headers: `X-Api-Key`
+   - Query Parameters:
+     - `limit` (optional): Number of results per page (default: 100)
+     - `cursor` (optional): Pagination cursor from previous response
+   - Response Example
+     ```json
+     {
+       "data": [
+         {
+           "id": "730e2202-58f9-478c-a24c-f1c561df6716",
+           "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0",
+           "country": "DE",
+           "browser": "Firefox",
+           "os": "Mac OS X",
+           "referer": "Direct",
+           "created_at": "2024-07-12T19:25:22Z"
+         }
+       ],
+       "pagination": {
+         "has_more": true,
+         "next": "629e3301-47f8-389b-b24c-f1c561df9825"
+       }
+     }
+     ```

-   - Endpoint\*\*: `POST /api/links`
+6. **Create new link**
+   - Endpoint: `POST /api/links`
   - Payload:
     ```json
     {
@@ -110,14 +109,13 @@
       "data": {
         "id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db",
         "refer": "http://localhost:4000/3wP4BQ",
-         "origin": "https://monocuco.donado.co/test",
+         "origin": "https://example.com",
         "clicks": []
       }
     }
     ```

-6. **Update an existing link by ID**
-
+7. **Update an existing link by ID**
   - Endpoint: `PUT /api/links/:id`
   - Payload:
     ```json
@@ -138,15 +136,8 @@
     }
     ```

-7. **Delete a link by ID**
-
-   - Endpoint: `DELETE /api/links/:id`
-   - Payload: None
-   - Headers: `X-Api-Key`
-   - Response Example:
-     ```json
-     {
-       "message": "Link deleted"
-     }
-     ```
-
+8. **Delete a link by ID**
+  - Endpoint: `DELETE /api/links/:id`
+  - Payload: None
+  - Headers: `X-Api-Key`
+  - Response: 204
@@ -99,21 +99,28 @@ ENV=test crystal spec

 ## Benchmark

-Conducted on a MacBook Air M2 with 16GB RAM.
+CPU: Apple M3 Pro

 ```
-$ ./benchmark.sh
+> colima start --cpu 1 --memory 1
+INFO[0000] starting colima
+INFO[0000] runtime: docker
+INFO[0001] starting ...                                  context=vm
+INFO[0076] provisioning ...                              context=docker
+INFO[0077] starting ...                                  context=docker
+INFO[0077] done
+
+> ./benchmark.sh
 Setting up...
-[+] Running 3/3
- ✔ Network bit_default       Created                                                                         0.0s
- ✔ Volume "bit_sqlite_data"  Created                                                                         0.0s
- ✔ Container bit             Started                                                                         0.1s
-Captured API Key: aHOCnZSuo2kOHy2mDa-iOA
+[+] Running 2/2
+ ✔ Network bit_default  Created                                                                                                                                                                     0.0s
+ ✔ Container bit        Started                                                                                                                                                                     0.1s
+Captured API Key: v-8gljT0WjMhQECito3e5g
 Waiting for the application to be ready...
 HTTP/1.1 200 OK
 Connection: keep-alive
 Content-Type: application/json
-Date: Sun, 27 Oct 2024 11:52:33 GMT
+Date: Sun, 16 Mar 2025 10:51:22 GMT
 Access-Control-Allow-Origin: *
 Access-Control-Allow-Methods: GET, POST, PUT, DELETE, OPTIONS
 Access-Control-Allow-Headers: Content-Type, Accept, Origin, X-Api-Key
@@ -123,25 +130,25 @@ Starting resource usage monitoring...
 Creating 10000 short links with 100 conrurrent requests...
 Link creation complete: 10000 links created.
 Fetching all created links from /api/links...
-Selected link for benchmarking: http://localhost:4000/UaVZjA
+Selected link for benchmarking: http://localhost:4000/pKtTjA
 Starting benchmark with Bombardier...
-Bombarding http://localhost:4000/oEKLAg with 10000 request(s) using 100 connection(s)
- 10000 / 10000 [===============================================================================] 100.00% 830/s 12s
+Bombarding http://localhost:4000/pKtTjA with 10000 request(s) using 100 connection(s)
+ 10000 / 10000 [======================================================================================================================================================================] 100.00% 1424/s 7s
 Done!
 Statistics        Avg      Stdev        Max
-  Reqs/sec       853.89    1625.49    8942.54
-  Latency      118.48ms    11.52ms   142.58ms
+  Reqs/sec      1885.24    7686.34  140641.16
+  Latency       68.00ms     6.43ms    89.56ms
  HTTP codes:
    1xx - 0, 2xx - 0, 3xx - 10000, 4xx - 0, 5xx - 0
    others - 0
-  Throughput:   360.02KB/s
+  Throughput:   625.54KB/s
 Benchmark completed.
 Analyzing resource usage...
 **** Results ****
-Average CPU Usage: 40.68%
-Average Memory Usage: 28.62 MiB
-./benchmark.sh: line 135: 61567 Terminated: 15          monitor_resource_usage
+Average CPU Usage: 42.98%
+Average Memory Usage: 33.25 MiB
+./benchmark.sh: line 135:  1500 Terminated: 15          monitor_resource_usage
 [+] Running 2/2
- ✔ Container bit        Removed                                                                                                                                                                                               10.1s
- ✔ Network bit_default  Removed  
+ ✔ Container bit        Removed                                                                                                                                                                    10.1s
+ ✔ Network bit_default  Removed                                                                                                                                                                     0.0s
 ```
@@ -19,11 +19,20 @@ OptionParser.parse do |parser|
    exit
  end

+  parser.on("--update-data", "Download all required data files (UA Parser and GeoLite2)") do
+    puts "=== Starting data files update ==="
+    App::Services::Cli.update_uap_regexes
+    App::Services::Cli.download_geolite_db
+    puts "=== All data files updated successfully ==="
+    exit
+  end
+
  if ARGV.empty?
    puts "Usage: ./cli [options]"
    puts "Options:"
-    puts "  --create-user=NAME  Create a new user with the given name"
-    puts "  --list-users        List all users"
-    puts "  --delete-user=USER_ID Delete a user by ID"
+    puts "  --create-user=NAME     Create a new user with the given name"
+    puts "  --list-users           List all users"
+    puts "  --delete-user=USER_ID  Delete a user by ID"
+    puts "  --update-data          Download all required data files"
  end
 end
@@ -20,10 +20,18 @@ shards:
    git: https://github.com/crystal-loot/exception_page.git
    version: 0.4.1

+  ipaddress:
+    git: https://github.com/sija/ipaddress.cr.git
+    version: 0.2.3
+
  kemal:
    git: https://github.com/kemalcr/kemal.git
    version: 1.5.0

+  maxminddb:
+    git: https://github.com/delef/maxminddb.cr.git
+    version: 1.5.0
+
  micrate:
    git: https://github.com/amberframework/micrate.git
    version: 0.15.1
@@ -1,5 +1,5 @@
 name: bit
-version: 1.4.1
+version: 1.5.1

 authors:
  - Juan Rodriguez <sjdonado@icloud.com>
@@ -22,7 +22,8 @@ dependencies:
    version: 0.15.1
  user_agent_parser:
    github: busyloop/user_agent_parser
-    version: 2.0.1
+  maxminddb:
+    github: delef/maxminddb.cr

 development_dependencies:
  dotenv:
@@ -82,33 +82,71 @@ describe "App::Controllers::Link" do
  end

  describe "Index" do
-    it "should redirect to origin domain" do
+    it "should redirect to origin domain with forwarded headers" do
      link = "https://test.com"
      test_user = create_test_user()

      test_link = create_test_link(test_user, link)
      serialized_link = App::Serializers::Link.new(test_link)

-      get(serialized_link.refer, headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
+      user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0"
+
+      get(serialized_link.refer, headers: HTTP::Headers{
+        "X-Api-Key" => test_user.api_key.to_s,
+        "User-Agent" => user_agent
+      })

      response.headers["Location"].should eq(link)
+      response.headers["User-Agent"].should eq(user_agent)
+      response.headers.has_key?("X-Forwarded-For").should be_true
    end

-    it "should create a new click after redirect" do
+    it "should create a new click after redirect with proper information" do
      link = "https://sjdonado.com"
      test_user = create_test_user()

      test_link = create_test_link(test_user, link)
      serialized_link = App::Serializers::Link.new(test_link)

-      get(serialized_link.refer, headers: HTTP::Headers{"User-Agent" => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0"})
+      user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0"
+      referer = "https://example.com/page"
+
+      get(serialized_link.refer, headers: HTTP::Headers{
+        "User-Agent" => user_agent,
+        "Referer" => referer
+      })

      Fiber.yield # replace yield with sleep 5 to debug errors

      response.headers["Location"].should eq(link)

+      # Verify that the click was recorded
      updated_test_link = get_test_link(test_link.id)
      updated_test_link.clicks.size.should eq(test_link.clicks.size + 1)
+
+      # Verify click details
+      latest_click = updated_test_link.clicks.last
+      latest_click.user_agent.should eq(user_agent)
+      latest_click.browser.should eq("Firefox")
+      latest_click.os.should eq("Mac OS X")
+      latest_click.referer.should eq("example.com") # Should extract host from the referer
+    end
+
+    it "should create a click with utm_source when no referer is provided" do
+      link = "https://sjdonado.com"
+      test_user = create_test_user()
+
+      test_link = create_test_link(test_user, link)
+      serialized_link = App::Serializers::Link.new(test_link)
+
+      # Add utm_source parameter
+      get("#{serialized_link.refer}?utm_source=email_campaign")
+
+      Fiber.yield
+
+      updated_test_link = get_test_link(test_link.id)
+      latest_click = updated_test_link.clicks.last
+      latest_click.referer.should eq("email_campaign")
    end

    it "should return 404 - link does not exist" do
@@ -123,8 +161,8 @@ describe "App::Controllers::Link" do
  end

  describe "All" do
-    it "should return all links" do
-      links = ["https://google.com", "google.com", "google.com.co"]
+    it "should return all links with pagination" do
+      links = ["https://sjdonado.com", "sjdonado.com", "sjdonado.com.co"]
      test_user = create_test_user()

      links.each do |link|
@@ -133,14 +171,58 @@ describe "App::Controllers::Link" do

      get("/api/links", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})

-      parsed_response = Hash(String, Array(Hash(String, String | Int64 | Array(Hash(String, String))))).from_json(response.body)
-      parsed_response["data"][0]["origin"].should eq(links[0])
-      parsed_response["data"][1]["origin"].should eq(links[1])
-      parsed_response["data"][2]["origin"].should eq(links[2])
+      parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body)
+
+      # Check that each link is in the response data
+      origins = parsed_response["data"].as(Array).map { |link| link["origin"] }
+      links.each do |link|
+        origins.should contain(link)
+      end
+
+      parsed_response["pagination"].as(Hash)["has_more"].should be_false
+    end
+
+    it "should respect custom limit parameter" do
+      test_user = create_test_user()
+
+      5.times do |i|
+        create_test_link(test_user, "https://example.com/#{i}")
+      end
+
+      get("/api/links?limit=2", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
+
+      parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body)
+      parsed_response["data"].as(Array).size.should eq(2)
+      parsed_response["pagination"].as(Hash)["has_more"].should be_true
+      parsed_response["pagination"].as(Hash)["next"].should_not be_nil
+    end
+
+    it "should support cursor-based pagination" do
+      test_user = create_test_user()
+
+      5.times do |i|
+        create_test_link(test_user, "https://example.com/#{i}")
+      end
+
+      # Get first page
+      get("/api/links?limit=2", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
+      first_page = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body)
+      cursor = first_page["pagination"].as(Hash)["next"]
+
+      # Get second page using cursor
+      get("/api/links?limit=2&cursor=#{cursor}", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
+      second_page = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body)
+
+      # Ensure different links are returned
+      first_page_ids = first_page["data"].as(Array).map { |link| link["id"] }
+      second_page_ids = second_page["data"].as(Array).map { |link| link["id"] }
+
+      # Check that no IDs from first page appear in second page
+      (first_page_ids & second_page_ids).empty?.should be_true
    end

    it "should return owned links only" do
-      links = ["https://google.de", "google.de", "google.edu.co", "x.com"]
+      links = ["https://donado.co", "donado.co", "uninorte.edu.co", "kagi.com"]
      test_user = create_test_user()

      links[0..2].each do |link|
@@ -152,11 +234,14 @@ describe "App::Controllers::Link" do

      get("/api/links", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})

-      parsed_response = Hash(String, Array(Hash(String, String | Int64 | Array(Hash(String, String))))).from_json(response.body)
-      parsed_response["data"].size.should eq(3)
-      parsed_response["data"][0]["origin"].should eq(links[0])
-      parsed_response["data"][1]["origin"].should eq(links[1])
-      parsed_response["data"][2]["origin"].should eq(links[2])
+      parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body)
+      parsed_response["data"].as(Array).size.should eq(3)
+
+      origins = parsed_response["data"].as(Array).map { |link| link["origin"] }
+      links[0..2].each do |link|
+        origins.should contain(link)
+      end
+      origins.should_not contain(links[3])
    end

    it "should return 401 - missing api key" do
@@ -169,16 +254,20 @@ describe "App::Controllers::Link" do
  end

  describe "Get" do
-    it "should return the specified link with click details" do
+    it "should return the specified link with limited click details" do
      link = "https://bing.com"
      test_user = create_test_user()
      test_link = create_test_link(test_user, link)

+      110.times do
+        create_test_click(test_link)
+      end
+
      get("/api/links/#{test_link.id}", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})

      parsed_response = Hash(String, Hash(String, String | Int64 | Array(Hash(String, String)))).from_json(response.body)
      parsed_response["data"]["origin"].should eq(link)
-      parsed_response["data"]["clicks"].should be_a(Array(Hash(String, String)))
+      parsed_response["data"]["clicks"].as(Array).size.should eq(100)
    end

    it "should return 404 - link does not exist" do
@@ -200,6 +289,85 @@ describe "App::Controllers::Link" do
    end
  end

+  describe "Clicks" do
+    it "should return paginated clicks for a link" do
+      link = "https://example.com"
+      test_user = create_test_user()
+      test_link = create_test_link(test_user, link)
+
+      5.times do
+        create_test_click(test_link)
+      end
+
+      get("/api/links/#{test_link.id}/clicks", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
+
+      parsed_response = Hash(String, Array(Hash(String, String)) | Hash(String, Bool | String?)).from_json(response.body)
+      parsed_response["data"].as(Array).size.should eq(5)
+      parsed_response["pagination"].as(Hash)["has_more"].should be_false
+    end
+
+    it "should respect limit parameter" do
+      link = "https://example.com"
+      test_user = create_test_user()
+      test_link = create_test_link(test_user, link)
+
+      10.times do
+        create_test_click(test_link)
+      end
+
+      get("/api/links/#{test_link.id}/clicks?limit=3", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
+
+      parsed_response = Hash(String, Array(Hash(String, String)) | Hash(String, Bool | String?)).from_json(response.body)
+      parsed_response["data"].as(Array).size.should eq(3)
+      parsed_response["pagination"].as(Hash)["has_more"].should be_true
+      parsed_response["pagination"].as(Hash)["next"].should_not be_nil
+    end
+
+    it "should support cursor-based pagination" do
+      link = "https://example.com"
+      test_user = create_test_user()
+      test_link = create_test_link(test_user, link)
+
+      10.times do
+        create_test_click(test_link)
+      end
+
+      # Get first page
+      get("/api/links/#{test_link.id}/clicks?limit=3", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
+      first_page = Hash(String, Array(Hash(String, String)) | Hash(String, Bool | String?)).from_json(response.body)
+      cursor = first_page["pagination"].as(Hash)["next"]
+
+      # Get second page using cursor
+      get("/api/links/#{test_link.id}/clicks?limit=3&cursor=#{cursor}", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
+      second_page = Hash(String, Array(Hash(String, String)) | Hash(String, Bool | String?)).from_json(response.body)
+
+      # Ensure different clicks are returned
+      first_page_ids = first_page["data"].as(Array).map { |click| click["id"] }
+      second_page_ids = second_page["data"].as(Array).map { |click| click["id"] }
+
+      # Check that no IDs from first page appear in second page
+      (first_page_ids & second_page_ids).empty?.should be_true
+    end
+
+    it "should return 404 - link does not exist" do
+      test_user = create_test_user()
+
+      get("/api/links/nonexistent_id/clicks", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
+
+      expected = {"error" => "Resource not found"}.to_json
+      response.status_code.should eq(404)
+      response.body.should eq(expected)
+    end
+
+    it "should return 401 - missing api key" do
+      get("/api/links/1/clicks")
+
+      expected = {"error" => "Unauthorized access"}.to_json
+      response.status_code.should eq(401)
+      response.body.should eq(expected)
+    end
+  end
+
  describe "Update" do
    it "should update link url" do
      link = "https://github.com"
@@ -4,7 +4,7 @@ describe "App::Controllers::Ping" do
  it "should return pong" do
    get "/api/ping"

-    expected = {"pong" => "ok"}.to_json
+    expected = {"data" => "pong"}.to_json
    response.body.should eq(expected)
  end
 end
@@ -55,6 +55,26 @@ def create_test_link(user, url)
  link
 end

+def create_test_click(link)
+  click = App::Models::Click.new
+  click.id = UUID.v4.to_s
+  click.user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0"
+  click.browser = "Firefox"
+  click.os = "Mac OS X"
+  click.referer = "example.com"
+  click.country = "US"
+  click.created_at = Time.utc
+  click.link = link
+  click.link_id = link.id
+
+  changeset = App::Lib::Database.insert(click)
+  unless changeset.valid?
+    error_messages = changeset.errors.map { |error| "#{error}" }.join(", ")
+    raise "Test click creation failed: #{error_messages}"
+  end
+  click
+end
+
 def get_test_link(link_id)
  query = App::Lib::Database::Query.where(id: link_id.as(String)).limit(1)
  link = App::Lib::Database.all(App::Models::Link, query, preload: [:clicks]).first?
Author	SHA1	Message	Date
sjdonado	21f53f257c	chore: upgrade version to 1.5.1	2025-03-16 18:56:33 +01:00
sjdonado	8ca6a450a3	chore: update API documentation	2025-03-16 18:55:18 +01:00
sjdonado	58d8d52194	test: update cursor-based pagination test cases	2025-03-16 18:55:02 +01:00
sjdonado	7d617bbb30	feat: api links/:id/clicks endpoint	2025-03-16 18:34:46 +01:00
sjdonado	cd6dfa345b	feat: links all cursor pagination	2025-03-16 18:30:52 +01:00
sjdonado	1967cc2c22	fix: get remote address cloudfare proxy	2025-03-16 18:04:36 +01:00
sjdonado	60ebac7150	chore: update API docs	2025-03-16 14:22:32 +01:00
sjdonado	a066b5e5ab	chore: upgrade version to 1.5.0	2025-03-16 13:41:54 +01:00
sjdonado	cd8e2433a5	feat(cli): --update-data download UA Parser and GeoLite2	2025-03-16 13:40:33 +01:00
sjdonado	b538a379d1	chore: update README	2025-03-16 11:54:48 +01:00
sjdonado	8cab7a51ad	feat: country ip lookup	2025-03-16 11:42:01 +01:00
sjdonado	ece74226d4	feat: add country to clicks	2025-03-16 11:26:20 +01:00
sjdonado	d26aa2f18a	feat: links redirect forward client ip	2025-03-16 10:20:01 +01:00
sjdonado	ce2f73dfe3	fix(ci): release pipeline determine version	2025-03-16 10:18:51 +01:00