chore: update parsers 04.2025

- return tuple directly from the block
- avoid to parse remote_address in the main thread
- avoid to replace headers, add only
- avoid separate variables for one-single use

.dockerignore

@@ -3,3 +3,7 @@
 /.shards/
 /spec/
 /sqlite/
+
+/docs/
+benchmark.cr
+.env*

.github/workflows/publish.yml

@@ -31,11 +31,14 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Extract version
+      - name: Determine version
         id: version
         run: |
-          VERSION=$(grep '^version:' shard.yml | cut -d ' ' -f 2)
-          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          if [ "${{ github.event_name }}" = "release" ]; then
+            echo "version=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT
+          else
+            echo "version=latest" >> $GITHUB_OUTPUT
+          fi
 
       - name: Build and push platform image
         uses: docker/build-push-action@v5
@@ -46,7 +49,7 @@ jobs:
           platforms: ${{ matrix.platform }}
           push: true
           tags: |
-            sjdonado/bit:${{ github.event_name == 'release' && steps.version.outputs.version || 'latest' }}-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
+            sjdonado/bit:${{ steps.version.outputs.version }}-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
           build-args: |
             TARGETARCH=${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
           cache-from: type=gha
@@ -65,8 +68,17 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
+      - name: Determine version
+        id: version
+        run: |
+          if [ "${{ github.event_name }}" = "release" ]; then
+            echo "version=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT
+          else
+            echo "version=latest" >> $GITHUB_OUTPUT
+          fi
+
       - name: Create manifest
         run: |
           docker buildx imagetools create \
-            -t sjdonado/bit:${{ github.event_name == 'release' && needs.build-platforms.outputs.version || 'latest' }} \
-            sjdonado/bit:${{ github.event_name == 'release' && needs.build-platforms.outputs.version || 'latest' }}-{amd64,arm64}
+            -t sjdonado/bit:${{ steps.version.outputs.version }} \
+            sjdonado/bit:${{ steps.version.outputs.version }}-{amd64,arm64}

.gitignore

@@ -7,4 +7,4 @@
 /sqlite/
 .env.production
 
-resource_usage.txt
+resource_usage.*

Dockerfile

@@ -33,7 +33,7 @@ WORKDIR /usr/src/app
 RUN apt-get update && apt-get install -y \
     libssl3 \
     libyaml-0-2 \
-    libsqlite3-0 \
+    sqlite3 \
     libevent-2.1-7 \
     && rm -rf /var/lib/apt/lists/*
 

README.md

@@ -2,27 +2,30 @@
 [![Docker Stars](https://img.shields.io/docker/stars/sjdonado/bit.svg)](https://hub.docker.com/r/sjdonado/bit)
 [![Docker Image Size](https://img.shields.io/docker/image-size/sjdonado/bit/latest)](https://hub.docker.com/r/sjdonado/bit)
 
-# Bit URL Shortener
+Lightweight URL shortener (API-only) with minimal resource requirements. Avg memory consumption **30MiB**, avg CPU load 20%.
 
-Lightweight URL shortener service with minimal resource requirements. Average memory consumption is **20MB RAM** with container disk space under **50MB**.
+Highly performant: **7.9k req/sec**, latency 15.8ms (100k requests using 125 connections, [benchmark](docs/SETUP.md#benchmark)).
 
-Bit is highly performant, achieving over 850 requests per second with an average latency of just 118ms. For detailed benchmark results, see [benchmark](docs/SETUP.md#benchmark).
+Self-hosted: [Dokku](docs/SETUP.md#dokku), [Docker Compose](docs/SETUP.md#docker-compose).
 
 Images available on [Docker Hub](https://hub.docker.com/r/sjdonado/bit/tags).
 
-## Quick Start
-```bash
-docker run -p 4000:4000 -e ADMIN_API_KEY=$(openssl rand -base64 32) sjdonado/bit:latest
-```
+## Why bit?
+It is feature-complete by design: simple and reliable without unnecessary bloat. Bug fixes will continue, but new features aren't planned.
 
-## Minimum Requirements
-- 50MB disk space
-- 50MB RAM (20MB avg usage)
-- x86_64 or ARM64 architecture
+- Minimal tracking setup: Country, browser, OS, referer. No cookies or persistent tracking mechanisms are used beyond what's available from a basic client's request.
+- Includes `X-Forwarded-For` header.
+- Multiple users are supported via API key authentication. Users can create, list and delete keys via the [CLI](docs/SETUP.md#cli).
+- Easy to extend, Ruby on Rails-inspired setup.
+
+## Recommented requirements
+- 100MB disk space
+- 70MiB RAM
+- x86_64 or ARM64
 
 ## Documentation
 - [API Reference](docs/API.md)
-- [Advanced Setup](docs/SETUP.md)
+- [Setup](docs/SETUP.md)
 
 ## Contributing
 Found an issue or have a suggestion? Please follow our [contribution guidelines](CONTRIBUTING.md).

app/controllers/click.cr

@@ -0,0 +1,118 @@
+module App::Controllers
+  struct ClickController
+    include App::Models
+    include App::Lib
+    include App::Services
+
+    # Buffered channel to hold click data
+    @@click_channel = Channel(NamedTuple(
+      link_id: Int64,
+      remote_address: String,
+      user_agent: String?,
+      referer: String
+    )).new(1024)
+
+    @@processor_started = begin
+      spawn do
+        batch = [] of NamedTuple(
+          link_id: Int64,
+          remote_address: String,
+          user_agent: String?,
+          referer: String
+        )
+
+        batch_size = 32
+        min_batch = 32
+        max_batch = 512
+        scaling_factor = 1.5
+
+        loop do
+          select
+          when click_data = @@click_channel.receive
+            batch << click_data
+
+            if batch.size >= batch_size
+              process_click_batch(batch)
+              batch.clear
+              # Increase batch size when reaching capacity
+              batch_size = (batch_size * scaling_factor).to_i.clamp(min_batch, max_batch)
+            end
+          when timeout(1.seconds)
+            unless batch.empty?
+              process_click_batch(batch)
+              batch.clear
+              batch_size = (batch_size / scaling_factor).to_i.clamp(min_batch, max_batch)
+            else
+              # Reset to default if idle
+              batch_size = min_batch unless batch_size == min_batch
+            end
+          end
+        end
+      end
+      true
+    end
+
+    private def self.process_click_batch(batch)
+      clicks = [] of App::Models::Click
+
+      batch.each do |click_data|
+        begin
+          client_ip = IpLookup.ip_from_address(click_data[:remote_address])
+          family, _, _, os = UserAgent.parse(click_data[:user_agent] || "")
+
+          click = App::Models::Click.new
+          click.link_id = click_data[:link_id]
+          click.country = client_ip ? IpLookup.country(client_ip) : nil
+          click.user_agent = click_data[:user_agent]
+          click.browser = family
+          click.os = os.try &.[0]  # OS family
+          click.referer = click_data[:referer]
+
+          clicks << click
+        rescue ex
+          Log.error { "Click data processing error: #{ex.message}" }
+        end
+      end
+
+      # Batch insert clicks if any were successfully processed
+       unless clicks.empty?
+        begin
+          multi = Crecto::Multi.new
+          clicks.each do |click|
+            multi.insert(click)
+          end
+          Database.transaction(multi)
+        rescue ex
+          Log.error { "Batch click insertion error: #{ex.message}" }
+        end
+      end
+    end
+
+    def self.redirect_handler
+      ->(env : HTTP::Server::Context) {
+        link_id, url = Database.raw_query("SELECT id, url FROM links WHERE slug = (?) LIMIT 1", env.params.url["slug"]) do |result|
+          result.move_next ? {result.read(Int64), result.read(String)} : nil
+        end || raise App::NotFoundException.new(env)
+
+        remote_address = env.request.headers["Cf-Connecting-Ip"]? || env.request.remote_address.to_s
+
+        env.response.status_code = 301
+        env.response.headers.add("Location", url)
+        env.response.headers.add("X-Forwarded-For", remote_address)
+
+        begin
+          @@click_channel.send({
+            link_id: link_id,
+            remote_address: remote_address,
+            user_agent: env.request.headers["User-Agent"]?,
+            referer: env.request.headers["Referer"]?.try { |r| URI.parse(r).host rescue r } || env.params.query["utm_source"]? || "Direct"
+          })
+        rescue Channel::ClosedError
+          Log.error { "Click channel closed" }
+        rescue ex
+          Log.error { "Error queuing click: #{ex.message}" }
+        end
+      }
+    end
+  end
+end

app/controllers/link.cr

@@ -1,179 +1,153 @@
-require "uuid"
-require "user_agent_parser"
-
-UserAgent.load_regexes(File.read("data/regexes.yaml"))
-
-require "../lib/controller.cr"
-
-module App::Controllers::Link
-  class Create < App::Lib::BaseController
+module App::Controllers
+  class LinkController < App::Lib::BaseController
     include App::Models
     include App::Lib
     include App::Services
 
-    def call(env)
-      user = env.get("user").as(User)
-      body = parse_body(env, ["url"])
+    def initialize(@env : HTTP::Server::Context)
+      super(@env)
+    end
+
+    def create
+      body = parse_body(["url"])
       url = body["url"].to_s
 
-      query = Database::Query.where(url: url, user_id: user.id.as(String)).limit(1)
+      query = Database::Query.where(url: url, user_id: current_user_id).limit(1)
       existing_link = Database.all(Link, query, preload: [:clicks]).first?
       if existing_link
-        response = {"data" => App::Serializers::Link.new(existing_link)}
-        return response.to_json
+        return render_json({"data" => App::Serializers::Link.new(existing_link)})
       end
 
       link = Link.new
-      link.id = UUID.v4.to_s
       link.url = url
-      link.user = user
-      link.slug = SlugService.shorten_url(url, user.id.to_s)
+      link.user_id = current_user_id
+      link.slug = SlugService.shorten_url(url, current_user_id)
 
       changeset = Database.insert(link)
       if !changeset.valid?
-        raise App::UnprocessableEntityException.new(env, map_changeset_errors(changeset.errors))
+        raise App::UnprocessableEntityException.new(@env, map_changeset_errors(changeset.errors))
       end
 
-      link.clicks = [] of App::Models::Click
-      response = {"data" => App::Serializers::Link.new(link)}
+      inserted_link = Database.get!(Link, changeset.instance.id)
 
-      response.to_json
-    end
+      render_json({"data" => App::Serializers::Link.new(inserted_link)}, 201)
     end
 
-  class Index < App::Lib::BaseController
-    include App::Models
-    include App::Lib
+    def list_all
+      limit, cursor = pagination_params
 
-    def call(env)
-      slug = env.params.url["slug"]
+      query = Database::Query.where(user_id: current_user_id)
+      query = query.where("id < ?", cursor) if cursor
+      query = query.order_by("id DESC").limit(limit + 1)
 
-      link = Database.get_by(Link, slug: slug)
-      raise App::NotFoundException.new(env) if !link
+      links = Database.all(Link, query)
 
-      spawn do
-        user_agent_str = env.request.headers["User-Agent"]? || "Unknown"
-        user_agent = user_agent_str != "Unknown" ? UserAgent.new(user_agent_str) : nil
-
-        language_header = env.request.headers["Accept-Language"]? || "Unknown"
-        language = language_header.split(',').first.split(';').first
-
-        referer = env.request.headers["Referer"]?
-
-        click = Click.new
-        click.id = UUID.v4.to_s
-        click.link = link
-        click.language = language
-        click.user_agent = user_agent_str
-        click.browser = user_agent ? user_agent.family : "Unknown"
-        click.os = user_agent ? (user_agent.os.try &.family || "Unknown") : "Unknown"
-        click.source = referer ? URI.parse(referer).host : "Unknown"
-
-        changeset = Database.insert(click)
-        if changeset.errors.any?
-          Log.error { "Logging click event failed: #{changeset.errors}" }
-        end
+      paginated_response(links, limit) { |link| App::Serializers::Link.new(link) }
     end
 
-      env.response.status_code = 301
-      env.response.headers["Location"] = link.url!
-      env.response.headers["Content-Type"] = "text/html"
-      env.response.print("Redirecting...")
-    end
+    def get
+      link_id = @env.params.url["id"].to_i64
+
+      query = Database::Query.where(id: link_id, user_id: current_user_id).limit(1)
+      link = Database.all(Link, query).first?
+      raise App::NotFoundException.new(@env) if link.nil?
+
+      clicks_query = Database::Query.where(link_id: link_id)
+                                   .order_by("id DESC")
+                                   .limit(100)
+      link.clicks = Database.all(Click, clicks_query)
+
+      render_json({"data" => App::Serializers::Link.new(link)})
     end
 
-  class All < App::Lib::BaseController
-    include App::Models
-    include App::Lib
+    def list_clicks
+      link_id = @env.params.url["id"].to_i64
 
-    def call(env)
-      user = env.get("user").as(User)
+      # Verify link exists and belongs to user
+      link_query = Database::Query.where(id: link_id, user_id: current_user_id).limit(1)
+      link = Database.all(Link, link_query).first?
+      raise App::NotFoundException.new(@env) if link.nil?
 
-      query = Database::Query.where(user_id: user.id.as(String))
-      links = Database.all(Link, query, preload: [:clicks])
+      limit, cursor = pagination_params
 
-      response = {"data" => links.map { |link| App::Serializers::Link.new(link) }}
-      response.to_json
-    end
+      query = Database::Query.where(link_id: link_id)
+      query = query.where("id < ?", cursor) if cursor
+      query = query.order_by("id DESC").limit(limit + 1)
+
+      clicks = Database.all(Click, query)
+
+      paginated_response(clicks, limit) { |click| App::Serializers::Click.new(click) }
     end
 
-  class Get < App::Lib::BaseController
-    include App::Models
-    include App::Lib
-
-    def call(env)
-      user = env.get("user").as(User)
-      link_id = env.params.url["id"]
-
-      query = Database::Query.where(id: link_id.as(String), user_id: user.id.as(String)).limit(1)
-      link = Database.all(Link, query, preload: [:clicks]).first?
-
-      raise App::NotFoundException.new(env) if link.nil?
-
-      response = {"data" => App::Serializers::Link.new(link)}
-      response.to_json
-    end
-  end
-
-  class Update < App::Lib::BaseController
-    include App::Models
-    include App::Lib
-    include App::Services
-
-    def call(env)
-      user = env.get("user").as(User)
-      id = env.params.url["id"]
-      body = parse_body(env, ["url"])
+    def update
+      id = @env.params.url["id"].to_i64
+      body = parse_body(["url"])
+      new_url = body["url"].to_s
 
       query = Database::Query.where(id: id).limit(1)
       link = Database.all(Link, query, preload: [:clicks]).first?
 
-      raise App::NotFoundException.new(env) if link.nil?
-      raise App::ForbiddenException.new(env) if link.user_id != user.id
+      raise App::NotFoundException.new(@env) if link.nil?
+      raise App::ForbiddenException.new(@env) if link.user_id != current_user_id
 
-      new_url = body["url"].to_s
-
-      existing_query = Database::Query.where(url: new_url, user_id: user.id.to_s).limit(1)
-      existing_link = Database.all(Link, existing_query).first?
-
-      if existing_link
-        raise App::UnprocessableEntityException.new(env, { "url" => ["URL already exists"] })
+      # Check for existing URL
+      existing_query = Database::Query.where(url: new_url, user_id: current_user_id).limit(1)
+      if Database.all(Link, existing_query).first?
+        raise App::UnprocessableEntityException.new(@env, { "url" => ["URL already exists"] })
       end
 
       link.url = new_url
-      link.slug = SlugService.shorten_url(new_url, user.id.to_s)
+      link.slug = SlugService.shorten_url(new_url, current_user_id)
 
       changeset = Database.update(link)
       if !changeset.valid?
-        raise App::UnprocessableEntityException.new(env, map_changeset_errors(changeset.errors))
+        raise App::UnprocessableEntityException.new(@env, map_changeset_errors(changeset.errors))
       end
 
-      response = {"data" => App::Serializers::Link.new(link)}
-      response.to_json
-    end
+      render_json({"data" => App::Serializers::Link.new(link)})
     end
 
-  class Delete < App::Lib::BaseController
-    include App::Models
-    include App::Lib
-
-    def call(env)
-      user = env.get("user").as(User)
-      id = env.params.url["id"]
+    def delete
+      id = @env.params.url["id"].to_i64
 
       link = Database.get(Link, id)
-      raise App::NotFoundException.new(env) if !link
+      raise App::NotFoundException.new(@env) if !link
+      raise App::ForbiddenException.new(@env) if link.user_id != current_user_id
 
-      if link.user_id != user.id
-        raise App::ForbiddenException.new(env)
-      end
-
-      result = Database.raw_exec("DELETE FROM links WHERE id = (?)", link.id) # tempfix: Database.delete does not work
+      result = Database.raw_exec("DELETE FROM links WHERE id = (?)", link.id)
       if result.rows_affected == 0
-        raise App::UnprocessableEntityException.new(env, { "id" => ["Row delete failed"] })
+        raise App::UnprocessableEntityException.new(@env, { "id" => ["Row delete failed"] })
       end
 
-      env.response.status_code = 204
+      @env.response.status_code = 204
+    end
+
+    private def current_user : User
+      @env.get("user").as(User)
+    end
+
+    private def current_user_id : Int64
+      current_user.id.as(Int64)
+    end
+
+    private def pagination_params
+      limit = (@env.params.query["limit"]? || "100").to_i32
+      cursor = @env.params.query["cursor"]?
+      {limit, cursor}
+    end
+
+    private def paginated_response(items, limit)
+      has_more = items.size > limit
+      items = items[0...limit] if has_more
+      next_cursor = has_more ? items.last.id : nil
+
+      render_json({
+        "data" => items.map { |item| yield item },
+        "pagination" => {
+          "has_more" => has_more,
+          "next_cursor" => next_cursor
+        }
+      })
     end
   end
 end

app/controllers/ping.cr

@@ -1,10 +1,13 @@
 require "../lib/controller.cr"
 
-module App::Controllers::Ping
-  class Get < App::Lib::BaseController
-    def call(env)
-      response = {"pong" => "ok"}
-      response.to_json
+module App::Controllers
+  class PingController < App::Lib::BaseController
+    def initialize(@env : HTTP::Server::Context)
+      super(@env)
+    end
+
+    def ping
+      render_json({data: "pong"})
     end
   end
 end

app/lib/controller.cr

@@ -1,29 +1,45 @@
 module App::Lib
   abstract class BaseController
-    def map_changeset_errors(errors)
+    protected getter env : HTTP::Server::Context
+
+    def initialize(@env : HTTP::Server::Context); end
+
+    # Convert changeset errors to API-friendly format
+    protected def map_changeset_errors(errors)
       errors.reduce({} of String => Array(String)) do |memo, error|
-        memo[error[:field]] = memo[error[:field]]? || [] of String
-        memo[error[:field]] << error[:message]
+        field = error[:field].to_s
+        message = error[:message].to_s
+
+        memo[field] ||= [] of String
+        memo[field] << message
         memo
       end
     end
 
-    def parse_body(env, fields)
-      json_params = env.params.json.to_h
-      missing_fields = [] of String
+    protected def parse_body(required_fields : Array(String) = [] of String)
+      json_params = @env.params.json.try(&.to_h) || {} of String => JSON::Any
+      json_params = json_params.transform_values(&.to_s) # Convert JSON::Any to String
 
-      fields.each do |field|
-        unless json_params.has_key?(field)
-          missing_fields << field
-        end
-      end
+      missing_fields = required_fields.reject { |field| json_params.has_key?(field) }
 
       unless missing_fields.empty?
-        error_message = missing_fields.map { |field| "#{field}: Required field" }.join(", ")
-        raise App::BadRequestException.new(env, error_message)
+        error_message = missing_fields.join(", ") + " required"
+        raise App::BadRequestException.new(@env, error_message)
       end
 
       json_params
     end
+
+    protected def render_json(data, status_code : Int32 = 200)
+      @env.response.status_code = status_code
+      @env.response.content_type = "application/json"
+      data.to_json
+    end
+
+    protected def param(key : String) : String
+      @env.params.url[key]
+    rescue KeyError
+      raise App::BadRequestException.new(@env, "Missing required parameter: #{key}")
+    end
   end
 end

app/lib/database.cr

@@ -1,6 +1,6 @@
 require "sqlite3"
 require "crecto"
-require"micrate"
+require "micrate"
 
 module App::Lib
   class Database
@@ -9,7 +9,15 @@ module App::Lib
     Query = Crecto::Repo::Query
 
     config do |conf|
-      conf.uri = ENV["DATABASE_URL"]
+      base_url = ENV["DATABASE_URL"]
+      separator = base_url.includes?("?") ? "&" : "?"
+
+      db_url = base_url + separator +
+        "&journal_mode=WAL" +
+        "&synchronous=NORMAL" +      # Better performance with reasonable safety
+        "&foreign_keys=true"
+
+      conf.uri = db_url
     end
 
     if ENV["ENV"] == "development"

app/lib/errors.cr

@@ -55,12 +55,3 @@ module App
     end
   end
 end
-
-error 500 do |env|
-  App::InternalServerErrorException.new(env)
-  ""
-end
-
-error 404 do |env|
-  ""
-end

app/lib/ip_lookup.cr

@@ -0,0 +1,45 @@
+require "maxminddb"
+require "log"
+
+module App::Lib
+  struct IpLookup
+    MMDB_PATH = "data/GeoLite2-Country.mmdb"
+
+    @@reader : MaxMindDB::Reader? = nil
+    @@reader_mutex = Mutex.new
+
+    private def self.get_reader : MaxMindDB::Reader
+      @@reader_mutex.synchronize do
+        @@reader ||= MaxMindDB.open(MMDB_PATH)
+      end
+    end
+
+    def self.country(ip_address : String) : String?
+      return nil if ip_address == "Unknown" || ip_address.empty?
+
+      begin
+        lookup = get_reader.get(ip_address)
+        lookup["country"]?.try &.["iso_code"]?.try &.as_s
+      rescue ex
+        Log.error { "IP lookup failed: #{ex.message}" }
+        nil
+      end
+    end
+
+    def self.ip_from_address(address_string : String?) : String?
+      return nil if address_string.nil?
+
+      if address_string.includes?('[') # IPv6 with port: [2001:db8::1]:8080
+        address_string.split(']').first.sub('[', '\'')
+      elsif address_string.includes?(':')
+        if address_string.count(':') > 1 # IPv6 without port
+          address_string
+        else # IPv4 with port: 192.168.1.1:8080
+          address_string.split(':').first
+        end
+      else # Address without port
+        address_string
+      end
+    end
+  end
+end

app/lib/ua_parser.cr

@@ -0,0 +1,123 @@
+require "yaml"
+require "semantic_version"
+
+module App::Lib
+  struct UserAgent
+    REGEXES_PATH = "data/uap_core_regexes.yaml"
+
+    @@regexes_cache : YAML::Any? = nil
+    @@compiled_regexes = {} of String => Array(Tuple(Regex, YAML::Any))
+    @@mutex = Mutex.new
+
+    private def self.load_regexes
+      @@mutex.synchronize do
+        if @@regexes_cache.nil?
+          begin
+            regexes_yaml = File.read(REGEXES_PATH)
+            @@regexes_cache = YAML.parse(regexes_yaml)
+
+            # Pre-compile all regexes for better performance
+            ["user_agent_parsers", "os_parsers", "device_parsers"].each do |parser_type|
+              @@compiled_regexes[parser_type] = [] of Tuple(Regex, YAML::Any)
+
+              @@regexes_cache.not_nil![parser_type].as_a.each do |parser|
+                regex_str = parser["regex"].as_s
+                options = parser["regex_flag"]?.try(&.as_s) == "i" ?
+                  Regex::Options::IGNORE_CASE : Regex::Options::None
+
+                begin
+                  compiled_regex = Regex.new(regex_str, options)
+                  @@compiled_regexes[parser_type] << {compiled_regex, parser}
+                rescue
+                  # Skip invalid regexes
+                end
+              end
+            end
+          rescue ex
+            # If loading fails, set an empty cache to prevent repeated failures
+            @@regexes_cache = YAML.parse("{}")
+            @@compiled_regexes = {} of String => Array(Tuple(Regex, YAML::Any))
+          end
+        end
+      end
+    end
+
+    def self.parse(user_agent_string : String)
+      return {nil, nil, nil, nil} if user_agent_string.empty?
+
+      # Load regexes only once and cache them
+      load_regexes
+
+      family = nil
+      version = nil
+      device = nil
+      os = nil
+
+      @@compiled_regexes["user_agent_parsers"]?.try &.each do |regex_tuple|
+        regex, parser = regex_tuple
+        match = regex.match(user_agent_string)
+        next unless match
+
+        family = match[1]? || nil
+        v1 = (match[2]? || "0").to_i
+        v2 = (match[3]? || "0").to_i
+        v3 = (match[4]? || "0").to_i
+
+        # Apply replacements if defined
+        if replacement = parser["family_replacement"]?
+          family = replacement.as_s.gsub("$1", family.to_s)
+        end
+
+        version = SemanticVersion.new(v1, v2, v3)
+        break
+      end
+
+      @@compiled_regexes["os_parsers"]?.try &.each do |regex_tuple|
+        regex, parser = regex_tuple
+        match = regex.match(user_agent_string)
+        next unless match
+
+        os_family = match[1]? || nil
+        os_v1 = (match[2]? || "0").to_i
+        os_v2 = (match[3]? || "0").to_i
+        os_v3 = (match[4]? || "0").to_i
+
+        # Apply replacements if defined
+        if replacement = parser["os_replacement"]?
+          os_family = replacement.as_s.gsub("$1", os_family.to_s)
+        end
+
+        os = {os_family, SemanticVersion.new(os_v1, os_v2, os_v3)}
+        break
+      end
+
+      @@compiled_regexes["device_parsers"]?.try &.each do |regex_tuple|
+        regex, parser = regex_tuple
+        match = regex.match(user_agent_string)
+        next unless match
+
+        model = match[1]? || nil
+        device_name = model
+        brand = nil
+
+        # Apply replacements if defined
+        if device_replacement = parser["device_replacement"]?
+          device_name = device_replacement.as_s.gsub("$1", device_name.to_s)
+        end
+
+        if model_replacement = parser["model_replacement"]?
+          model = model_replacement.as_s.gsub("$1", model.to_s)
+        end
+
+        if brand_replacement = parser["brand_replacement"]?
+          brand = brand_replacement.as_s
+        end
+
+        device = {model, brand, device_name}
+        break
+      end
+
+      {family, version, device, os}
+    end
+  end
+end

app/middlewares/cors.cr

@@ -0,0 +1,29 @@
+module App::Middlewares
+  class CORSHandler < Kemal::Handler
+    exclude ["/api/ping", "/:slug"]
+
+    def initialize(
+      @allow_origin = "*",
+      @allow_methods = "GET, POST, PUT, DELETE, OPTIONS",
+      @allow_headers = "Content-Type, Accept, Origin, X-Api-Key"
+    )
+    end
+
+    def call(env)
+      return call_next(env) if exclude_match?(env)
+
+      env.response.headers["Access-Control-Allow-Origin"] = @allow_origin
+      env.response.headers["Access-Control-Allow-Methods"] = @allow_methods
+      env.response.headers["Access-Control-Allow-Headers"] = @allow_headers
+
+      if env.request.method == "OPTIONS"
+        env.response.status_code = 200
+        env.response.content_type = "text/plain"
+        env.response.print("")
+        return env
+      end
+
+      call_next(env)
+    end
+  end
+end

app/models/click.cr

@@ -3,16 +3,16 @@ require "crecto"
 module App::Models
   class Click < Crecto::Model
     schema :clicks do
-      field :id, String, primary_key: true
+      field :id, Int64, primary_key: true
       field :user_agent, String
-      field :language, String
+      field :country, String
       field :browser, String
       field :os, String
-      field :source, String
+      field :referer, String
 
       belongs_to :link, Link
     end
 
-    validate_required [:user_agent, :language, :source]
+    validate_required [:user_agent, :referer]
   end
 end

app/models/link.cr

@@ -6,7 +6,7 @@ require "./user.cr"
 module App::Models
   class Link < Crecto::Model
     schema :links do
-      field :id, String, primary_key: true
+      field :id, Int64, primary_key: true
       field :slug, String
       field :url, String
 

app/models/user.cr

@@ -4,7 +4,7 @@ require "crecto"
 module App::Models
   class User < Crecto::Model
     schema :users do
-      field :id, String, primary_key: true
+      field :id, Int64, primary_key: true
       field :name, String
       field :api_key, String
     end

app/routes.cr

@@ -1,41 +1,44 @@
 require "./controllers/**"
 
+require "kemal"
+
+add_handler App::Middlewares::CORSHandler.new
+add_handler App::Middlewares::Auth.new
+
 module App
-  before_all do |env|
-    env.response.headers["Access-Control-Allow-Origin"] = "*"
-    env.response.headers["Access-Control-Allow-Methods"] = "GET, POST, PUT, DELETE, OPTIONS"
-    env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Accept, Origin, X-Api-Key"
-  end
-
-  after_all do |env|
-    env.response.content_type = "application/json"
-  end
+  get "/:slug", &App::Controllers::ClickController.redirect_handler
 
+  # Namespace /api
   get "/api/ping" do |env|
-    Controllers::Ping::Get.new.call(env)
-  end
-
-  get "/:slug" do |env|
-    Controllers::Link::Index.new.call(env)
+    Controllers::PingController.new(env).ping
   end
 
   get "/api/links" do |env|
-    Controllers::Link::All.new.call(env)
+    Controllers::LinkController.new(env).list_all
   end
 
   get "/api/links/:id" do |env|
-    Controllers::Link::Get.new.call(env)
+    Controllers::LinkController.new(env).get
+  end
+
+  get "/api/links/:id/clicks" do |env|
+    Controllers::LinkController.new(env).list_clicks
   end
 
   post "/api/links" do |env|
-    Controllers::Link::Create.new.call(env)
+    Controllers::LinkController.new(env).create
   end
 
   put "/api/links/:id" do |env|
-    Controllers::Link::Update.new.call(env)
+    Controllers::LinkController.new(env).update
   end
 
   delete "/api/links/:id" do |env|
-    Controllers::Link::Delete.new.call(env)
+    Controllers::LinkController.new(env).delete
+  end
+
+  error 500 do |env|
+    App::InternalServerErrorException.new(env)
+    ""
   end
 end

app/serializers/click.cr

@@ -11,10 +11,10 @@ module App::Serializers
       builder.object do
         builder.field("id", @click.id)
         builder.field("user_agent", @click.user_agent)
-        builder.field("language", @click.language)
+        builder.field("country", @click.country)
         builder.field("browser", @click.browser)
         builder.field("os", @click.os)
-        builder.field("source", @click.source)
+        builder.field("referer", @click.referer)
         builder.field("created_at", @click.created_at)
       end
     end

app/serializers/link.cr

@@ -16,7 +16,15 @@ module App::Serializers
         builder.field("id", @link.id)
         builder.field("refer", @refer)
         builder.field("origin", @link.url)
-        builder.field("clicks", @link.clicks.map { |click| App::Serializers::Click.new(click) })
+
+        begin
+          clicks = @link.clicks
+          unless clicks.empty?
+            builder.field("clicks", clicks.map { |click| App::Serializers::Click.new(click) })
+          end
+        rescue Crecto::AssociationNotLoaded
+          # Association not loaded, skip this field silently
+        end
       end
     end
   end

app/services/cli.cr

@@ -1,3 +1,6 @@
+require "file_utils"
+require "http/client"
+
 require "../config/*"
 require "../lib/*"
 require "../models/*"
@@ -5,12 +8,11 @@ require "../models/*"
 module App::Services::Cli
   def self.create_user(name, api_key = nil)
     user = App::Models::User.new
-    user.id = UUID.v4.to_s
     user.name = name
     user.api_key = api_key || Random::Secure.urlsafe_base64()
 
     changeset = App::Lib::Database.insert(user)
-    return changeset.errors if !changeset.valid?
+    return changeset.errors unless changeset.valid?
 
     "New user created: Name: #{user.name}, X-Api-Key: #{user.api_key}"
   end
@@ -53,4 +55,86 @@ module App::Services::Cli
       puts "Admin setup skipped: Missing ADMIN_NAME or ADMIN_API_KEY environment variables."
     end
   end
+
+  def self.update_uap_regexes
+    puts "Downloading User-Agent Parser core regexes..."
+
+    FileUtils.mkdir_p("data")
+    url = "https://raw.githubusercontent.com/ua-parser/uap-core/master/regexes.yaml"
+    output_file = "data/uap_core_regexes.yaml"
+
+    begin
+      http_get_with_redirect(url) do |response|
+        File.write(output_file, response.body_io.gets_to_end)
+      end
+      puts "User-Agent regexes downloaded to #{output_file}"
+    rescue e
+      puts "Error: Failed to download UAP core regexes: #{e.message}"
+    end
+  end
+
+  def self.download_geolite_db
+    puts "Downloading GeoLite2 Country database..."
+
+    FileUtils.mkdir_p("data")
+    url = "https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-Country.mmdb"
+    output_file = "data/GeoLite2-Country.mmdb"
+
+    begin
+      File.open(output_file, "wb") do |file|
+        http_get_with_redirect(url) do |response|
+          IO.copy(response.body_io, file)
+        end
+      end
+      puts "GeoLite2 database downloaded to #{output_file}"
+    rescue e
+      puts "Error: Failed to download GeoLite2 database: #{e.message}"
+    end
+  end
+
+  private def self.http_get_with_redirect(url : String, max_redirects = 5)
+    redirects = 0
+
+    while redirects < max_redirects
+      uri = URI.parse(url)
+      client = HTTP::Client.new(uri)
+
+      success = false
+      follow_redirect = false
+      redirect_url = nil
+
+      begin
+        client.get(uri.request_target) do |response|
+          case response.status_code
+          when 200
+            yield response
+            success = true
+          when 301, 302
+            if new_location = response.headers["Location"]?
+              puts "Following redirect to: #{new_location}"
+              redirect_url = new_location
+              follow_redirect = true
+            else
+              raise "Received redirect status but no Location header"
+            end
+          else
+            raise "Failed request with status code: #{response.status_code}"
+          end
+        end
+      ensure
+        client.close
+      end
+
+      return if success
+
+      if follow_redirect && redirect_url
+        url = redirect_url
+        redirects += 1
+      else
+        break
+      end
+    end
+
+    raise "Too many redirects (#{max_redirects})"
+  end
 end

app/services/slug.cr

@@ -2,7 +2,7 @@ require "digest"
 require "base64"
 
 module App::Services::SlugService
-  def self.shorten_url(url : String, user_id : String) : String
+  def self.shorten_url(url : String, user_id : Int64) : String
     combined = "#{user_id}-#{url}"
     crc32_hash = Digest::CRC32.digest(combined)
     base62_encoded = Base64.urlsafe_encode(crc32_hash).strip.tr("-_=", "")

app/version.cr

@@ -1,3 +0,0 @@
-module App
-  VERSION = "0.1.0"
-end

benchmark.cr

@@ -0,0 +1,257 @@
+#!/usr/bin/env crystal
+
+require "http/client"
+require "json"
+require "file_utils"
+
+# Configuration variables
+SERVER_URL = "http://localhost:4000"
+API_URL = "#{SERVER_URL}/api/links"
+API_KEY = "secure_api_key_1"
+NUMBER_OF_REQUESTS = 100000
+
+CONTAINER_NAME = "bit"
+STATS_FILE = "resource_usage.txt"
+
+class ResourceMonitor
+  def initialize(@container_name : String)
+    @running = false
+    @stats = [] of {timestamp: Time, cpu: Float64, memory: Float64}
+  end
+
+  def start
+    @running = true
+    @stats.clear
+
+    # Initialize stats file with header
+    File.write(STATS_FILE, "Timestamp\tCPU(%)\tMemory(MiB)\n")
+
+    spawn do
+      while @running
+        if stat = capture_stats
+          # Append each measurement directly to the file
+          File.open(STATS_FILE, "a") do |file|
+            file.puts "#{stat[:timestamp].to_unix}\t#{stat[:cpu]}\t#{stat[:memory]}"
+          end
+          @stats << stat
+        end
+      end
+    end
+  end
+
+  def stop
+    @running = false
+  end
+
+  def avg_stats
+    return {cpu: 0.0, memory: 0.0} if @stats.empty?
+
+    total_cpu = 0.0
+    total_memory = 0.0
+    @stats.each do |stat|
+      total_cpu += stat[:cpu]
+      total_memory += stat[:memory]
+    end
+
+    {
+      cpu: total_cpu / @stats.size,
+      memory: total_memory / @stats.size
+    }
+  end
+
+  private def capture_stats
+    output = IO::Memory.new
+    process = Process.run(
+      "docker", ["stats", "--no-stream", "--format", "{{.CPUPerc}},{{.MemUsage}}", @container_name],
+      output: output
+    )
+
+    if process.success?
+      line = output.to_s.strip
+      parts = line.split(",")
+      if parts.size == 2
+        cpu_part = parts[0].gsub("%", "").to_f
+
+        # Extract the memory value properly by removing the "MiB" suffix
+        mem_string = parts[1].split.first
+        mem_part = mem_string.gsub(/[A-Za-z]+$/, "").to_f
+
+        return {timestamp: Time.utc, cpu: cpu_part, memory: mem_part}
+      end
+    end
+    nil
+  end
+end
+
+def check_dependencies
+  {"docker", "jq", "bombardier"}.each do |cmd|
+    process = Process.run("which", [cmd], output: Process::Redirect::Close)
+    unless process.success?
+      puts "Error: #{cmd} is not installed. Please install it to proceed."
+      exit(1)
+    end
+  end
+end
+
+def setup_containers
+  puts "Setting up..."
+
+  process = Process.run("docker", ["compose", "up", "-d"])
+  unless process.success?
+    puts "Failed to start Docker containers."
+    exit(1)
+  end
+
+  puts "Waiting for the application to be ready..."
+  until begin
+          HTTP::Client.get("#{SERVER_URL}/api/ping").success?
+        rescue
+          false
+        end
+    sleep 1.seconds
+  end
+
+  puts "Seeding the database..."
+  process = Process.run("docker", ["compose", "exec", "app", "sh", "-c", "sqlite3 ./sqlite/data.db < ./db/seed.sql"])
+
+  unless process.success?
+    puts "Error on seeding database"
+    exit(1)
+  end
+
+  puts "Checking seed results..."
+  until begin
+          HTTP::Client.get(
+            "#{API_URL}?limit=1",
+            headers: HTTP::Headers{"X-Api-Key" => API_KEY}
+          ).success?
+        rescue
+          false
+        end
+    sleep 2.seconds
+  end
+end
+
+def run_benchmark
+  puts "Fetching all created links from /api/links..."
+
+  response = HTTP::Client.get(
+    "#{API_URL}?limit=10000",
+    headers: HTTP::Headers{"X-Api-Key" => API_KEY}
+  )
+
+  sleep 2.seconds
+  unless response.success?
+    puts "Failed to fetch links. Status: #{response.status_code}"
+    exit(1)
+  end
+
+  data = JSON.parse(response.body)
+  links = data["data"].as_a.map { |link| link["refer"].as_s }
+
+  random_link = links.sample
+  puts "Selected link for benchmarking: #{random_link}"
+
+  puts "Starting benchmark with Bombardier..."
+
+  sleep 2.seconds
+  process = Process.new(
+    "bombardier",
+    ["-n", NUMBER_OF_REQUESTS.to_s, "-l", "--disableKeepAlives", random_link],
+    output: Process::Redirect::Inherit,
+    error: Process::Redirect::Inherit
+  )
+
+  status = process.wait
+
+  if status.success?
+    puts "Benchmark completed successfully."
+  else
+    puts "Bombardier failed with error code: #{status.exit_code}"
+    exit(1)
+  end
+end
+
+def analyze_resource_usage
+  puts "Analyzing resource usage..."
+
+  sleep 2.seconds
+  if File.exists?(STATS_FILE)
+    lines = File.read_lines(STATS_FILE)
+    # Skip header
+    lines = lines[1..-1] if lines.size > 0
+
+    if lines.size > 0
+      total_cpu = 0.0
+      total_memory = 0.0
+      peak_cpu = 0.0
+      peak_memory = 0.0
+
+      lines.each do |line|
+        fields = line.split("\t")
+        if fields.size >= 3
+          begin
+            cpu = fields[1].to_f
+            memory = fields[2].to_f
+
+            total_cpu += cpu
+            total_memory += memory
+
+            # Track peaks in a single pass
+            peak_cpu = cpu if cpu > peak_cpu
+            peak_memory = memory if memory > peak_memory
+          rescue
+            # Skip invalid lines
+          end
+        end
+      end
+
+      avg_cpu = total_cpu / lines.size
+      avg_memory = total_memory / lines.size
+
+      stats_summary = <<-STATS
+      **** Resource Usage Statistics ****
+        Measurements: #{lines.size}
+        Average CPU Usage: #{avg_cpu.round(2)}%
+        Average Memory Usage: #{avg_memory.round(2)} MiB
+        Peak CPU Usage: #{peak_cpu.round(2)}%
+        Peak Memory Usage: #{peak_memory.round(2)} MiB
+
+      STATS
+
+      File.open(STATS_FILE, "a") do |file|
+        file.puts "\n" + stats_summary
+      end
+
+      puts File.read(STATS_FILE)
+    else
+      puts "No resource usage data collected."
+    end
+  else
+    puts "Resource usage file not found."
+  end
+end
+
+def cleanup
+  Process.run("docker", ["compose", "down"])
+  puts "Cleanup completed. Resource usage data saved in #{STATS_FILE}"
+end
+
+def main
+  check_dependencies
+  setup_containers
+
+  monitor = ResourceMonitor.new(CONTAINER_NAME)
+  monitor.start
+
+  begin
+    run_benchmark
+
+    monitor.stop
+    analyze_resource_usage
+  ensure
+    cleanup
+  end
+end
+
+main

benchmark.sh

@@ -1,156 +0,0 @@
-#!/bin/bash
-
-# Configuration variables
-server_url="http://localhost:4000"
-api_url="${server_url}/api/links"
-num_links=10000
-num_requests=10000
-concurrency=100
-resource_usage_interval=1
-container_name="bit"
-
-check_dependencies() {
-    if ! command -v bombardier &> /dev/null; then
-        echo "Error: bombardier is not installed. Please install it to proceed."
-        exit 1
-    fi
-
-    if ! command -v jq &> /dev/null; then
-        echo "Error: jq is not installed. Please install it to proceed."
-        exit 1
-    fi
-}
-
-setup_containers() {
-    echo "Setting up..."
-    docker compose up -d
-    if [ $? -ne 0 ]; then
-        echo "Failed to start Docker containers."
-        exit 1
-    fi
-
-    output=$(docker compose exec -T app cli --create-user=Admin)
-    api_key=$(echo "$output" | awk -F' ' '/X-Api-Key:/{print $NF}')
-    echo "Captured API Key: $api_key"
-
-    if [[ -z "$api_key" ]]; then
-        echo "Error: API key could not be retrieved."
-        exit 1
-    fi
-
-    echo "Waiting for the application to be ready..."
-    until curl --silent --head --fail --header "X-Api-Key: $api_key" "$server_url/api/ping"; do
-        sleep 2
-    done
-}
-
-monitor_resource_usage() {
-    echo "Starting resource usage monitoring..."
-    echo "Timestamp,CPU,Memory" > resource_usage.csv
-    while :; do
-        stats=$(docker stats --no-stream --format "{{.CPUPerc}},{{.MemUsage}}" $container_name)
-        cpu=$(echo $stats | awk -F',' '{print $1}' | sed 's/%//')
-        mem=$(echo $stats | awk -F',' '{print $2}' | awk '{print $1}')
-        timestamp=$(date +%s)
-        echo "$timestamp,$cpu,$mem" >> resource_usage.csv
-        sleep $resource_usage_interval
-    done
-}
-
-create_links() {
-    local temp_file=$(mktemp)
-
-    echo "Creating $num_links short links with $concurrency conrurrent requests..."
-
-    # Populate URLs into a file to feed into curl
-    for ((i=1; i<=num_links; i++)); do
-        url="https://example.com/${i}-${num_links}"
-        echo "--next" >> "$temp_file"
-        echo "--request POST" >> "$temp_file"
-        echo "--url \"$api_url\"" >> "$temp_file"
-        echo "--header \"X-Api-Key: $api_key\"" >> "$temp_file"
-        echo "--header \"Content-Type: application/json\"" >> "$temp_file"
-        echo "--data \"{ \\\"url\\\": \\\"$url\\\" }\"" >> "$temp_file"
-    done
-
-    curl --parallel --parallel-immediate --parallel-max $concurrency --config "$temp_file" --silent --write-out "%{http_code}\n" > /dev/null
-
-    echo "Link creation complete: $num_links links created."
-
-    # Clean up
-    rm -f "$temp_file"
-}
-
-run_benchmark() {
-    echo "Fetching all created links from /api/links..."
-    all_links_response=$(curl --silent --request GET \
-                              --url "$api_url" \
-                              --header "X-Api-Key: $api_key" \
-                              --header "Content-Type: application/json")
-
-    links=($(echo "$all_links_response" | jq -r '.data[] | .refer'))
-    if [[ ${#links[@]} -ne $num_links ]]; then
-        echo "Error: Expected $num_links links but found ${#links[@]}."
-        exit 1
-    fi
-
-    random_link="${links[RANDOM % ${#links[@]}]}"
-    echo "Selected link for benchmarking: $random_link"
-
-    echo "Starting benchmark with Bombardier..."
-    bombardier -c $concurrency -n $num_requests "$random_link"
-    echo "Benchmark completed."
-}
-
-analyze_resource_usage() {
-    echo "Analyzing resource usage..."
-    total_cpu=0
-    total_mem=0
-    count=0
-
-    while IFS=',' read -r timestamp cpu mem; do
-        # Skip header line and lines with empty cpu or mem values
-        if [[ $timestamp != "Timestamp" && -n $cpu && -n $mem ]]; then
-            mem=${mem%MiB}
-
-            total_cpu=$(echo "$total_cpu + $cpu" | bc)
-            total_mem=$(echo "$total_mem + $mem" | bc)
-            ((count++))
-        fi
-    done < resource_usage.csv
-
-    avg_cpu=0.00
-    avg_mem=0.00
-
-    if (( count > 0 )); then
-        avg_cpu=$(echo "scale=2; $total_cpu / $count" | bc)
-        avg_mem=$(echo "scale=2; $total_mem / $count" | bc)
-    fi
-
-    echo "**** Results ****"
-    echo "Average CPU Usage: $avg_cpu%"
-    echo "Average Memory Usage: $avg_mem MiB"
-}
-
-cleanup() {
-    rm -f resource_usage.csv
-    docker compose down
-}
-
-main() {
-    check_dependencies
-    setup_containers
-
-    monitor_resource_usage &  # Start monitoring in the background
-    monitor_pid=$!
-    trap 'kill $monitor_pid; cleanup; exit' INT
-
-    create_links
-    run_benchmark
-
-    kill $monitor_pid
-    analyze_resource_usage
-    cleanup
-}
-
-main

bit.cr

@@ -10,8 +10,6 @@ require "./app/services/*"
 require "./app/routes"
 
 add_context_storage_type(App::Models::User)
-add_handler(App::Middlewares::Auth.new)
-
 App::Services::Cli.setup_admin_user
 
 Kemal.run

data/uap_core_regexes.yaml

@@ -148,11 +148,11 @@ user_agent_parsers:
     family_replacement: 'Pinterestbot'
 
   # Bots
-  - regex: '(CSimpleSpider|Cityreview Robot|CrawlDaddy|CrawlFire|Finderbots|Index crawler|Job Roboter|KiwiStatus Spider|Lijit Crawler|QuerySeekerSpider|ScollSpider|Trends Crawler|USyd-NLP-Spider|SiteCat Webbot|BotName\/\$BotVersion|123metaspider-Bot|1470\.net crawler|50\.nu|8bo Crawler Bot|Aboundex|Accoona-[A-z]{1,30}-Agent|AdsBot-Google(?:-[a-z]{1,30}|)|altavista|AppEngine-Google|archive.{0,30}\.org_bot|archiver|Ask Jeeves|[Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]{1,30})(?:-[A-Za-z]{1,30}|)|bingbot|BingPreview|blitzbot|BlogBridge|Bloglovin|BoardReader Blog Indexer|BoardReader Favicon Fetcher|boitho.com-dc|BotSeer|BUbiNG|\b\w{0,30}favicon\w{0,30}\b|\bYeti(?:-[a-z]{1,30}|)|Catchpoint(?: bot|)|[Cc]harlotte|Checklinks|clumboot|Comodo HTTP\(S\) Crawler|Comodo-Webinspector-Crawler|ConveraCrawler|CRAWL-E|CrawlConvera|Daumoa(?:-feedfetcher|)|Feed Seeker Bot|Feedbin|findlinks|Flamingo_SearchEngine|FollowSite Bot|furlbot|Genieo|gigabot|GomezAgent|gonzo1|(?:[a-zA-Z]{1,30}-|)Googlebot(?:-[a-zA-Z]{1,30}|)|Google SketchUp|grub-client|gsa-crawler|heritrix|HiddenMarket|holmes|HooWWWer|htdig|ia_archiver|ICC-Crawler|Icarus6j|ichiro(?:/mobile|)|IconSurf|IlTrovatore(?:-Setaccio|)|InfuzApp|Innovazion Crawler|InternetArchive|IP2[a-z]{1,30}Bot|jbot\b|KaloogaBot|Kraken|Kurzor|larbin|LEIA|LesnikBot|Linguee Bot|LinkAider|LinkedInBot|Lite Bot|Llaut|lycos|Mail\.RU_Bot|masscan|masidani_bot|Mediapartners-Google|Microsoft .{0,30} Bot|mogimogi|mozDex|MJ12bot|msnbot(?:-media {0,2}|)|msrbot|Mtps Feed Aggregation System|netresearch|Netvibes|NewsGator[^/]{0,30}|^NING|Nutch[^/]{0,30}|Nymesis|ObjectsSearch|OgScrper|Orbiter|OOZBOT|PagePeeker|PagesInventory|PaxleFramework|Peeplo Screenshot Bot|PHPCrawl|PlantyNet_WebRobot|Pompos|Qwantify|Read%20Later|Reaper|RedCarpet|Retreiver|Riddler|Rival IQ|scooter|Scrapy|Scrubby|searchsight|seekbot|semanticdiscovery|SemrushBot|Simpy|SimplePie|SEOstats|SimpleRSS|SiteCon|Slackbot-LinkExpanding|Slack-ImgProxy|Slurp|snappy|Speedy Spider|Squrl Java|Stringer|TheUsefulbot|ThumbShotsBot|Thumbshots\.ru|Tiny Tiny RSS|Twitterbot|WhatsApp|URL2PNG|Vagabondo|VoilaBot|^vortex|Votay bot|^voyager|WASALive.Bot|Web-sniffer|WebThumb|WeSEE:[A-z]{1,30}|WhatWeb|WIRE|WordPress|Wotbox|www\.almaden\.ibm\.com|Xenu(?:.s|) Link Sleuth|Xerka [A-z]{1,30}Bot|yacy(?:bot|)|YahooSeeker|Yahoo! Slurp|Yandex\w{1,30}|YodaoBot(?:-[A-z]{1,30}|)|YottaaMonitor|Yowedo|^Zao|^Zao-Crawler|ZeBot_www\.ze\.bz|ZooShot|ZyBorg|ArcGIS Hub Indexer)(?:[ /]v?(\d+)(?:\.(\d+)(?:\.(\d+)|)|)|)'
+  - regex: '(CSimpleSpider|Cityreview Robot|CrawlDaddy|CrawlFire|Finderbots|Index crawler|Job Roboter|KiwiStatus Spider|Lijit Crawler|QuerySeekerSpider|ScollSpider|Trends Crawler|USyd-NLP-Spider|SiteCat Webbot|BotName\/\$BotVersion|123metaspider-Bot|1470\.net crawler|50\.nu|8bo Crawler Bot|Aboundex|Accoona-[A-z]{1,30}-Agent|AdsBot-Google(?:-[a-z]{1,30}|)|altavista|AppEngine-Google|archive.{0,30}\.org_bot|archiver|Ask Jeeves|[Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]{1,30})(?:-[A-Za-z]{1,30}|)|bingbot|BingPreview|blitzbot|BlogBridge|Bloglovin|BoardReader Blog Indexer|BoardReader Favicon Fetcher|boitho.com-dc|BotSeer|BUbiNG|\b\w{0,30}favicon\w{0,30}\b|\bYeti(?:-[a-z]{1,30}|)|Catchpoint(?: bot|)|[Cc]harlotte|Checklinks|clumboot|Comodo HTTP\(S\) Crawler|Comodo-Webinspector-Crawler|ConveraCrawler|CRAWL-E|CrawlConvera|Daumoa(?:-feedfetcher|)|Feed Seeker Bot|Feedbin|findlinks|Flamingo_SearchEngine|FollowSite Bot|furlbot|Genieo|gigabot|GomezAgent|gonzo1|(?:[a-zA-Z]{1,30}-|)Googlebot(?:-[a-zA-Z]{1,30}|)|GoogleOther|Google SketchUp|grub-client|gsa-crawler|heritrix|HiddenMarket|holmes|HooWWWer|htdig|ia_archiver|ICC-Crawler|Icarus6j|ichiro(?:/mobile|)|IconSurf|IlTrovatore(?:-Setaccio|)|InfuzApp|Innovazion Crawler|InternetArchive|IP2[a-z]{1,30}Bot|jbot\b|KaloogaBot|Kraken|Kurzor|larbin|LEIA|LesnikBot|Linguee Bot|LinkAider|LinkedInBot|Lite Bot|Llaut|lycos|Mail\.RU_Bot|masscan|masidani_bot|Mediapartners-Google|Microsoft .{0,30} Bot|mogimogi|mozDex|MJ12bot|msnbot(?:-media {0,2}|)|msrbot|Mtps Feed Aggregation System|netresearch|Netvibes|NewsGator[^/]{0,30}|^NING|Nutch[^/]{0,30}|Nymesis|ObjectsSearch|OgScrper|Orbiter|OOZBOT|PagePeeker|PagesInventory|PaxleFramework|Peeplo Screenshot Bot|PHPCrawl|PlantyNet_WebRobot|Pompos|Qwantify|Read%20Later|Reaper|RedCarpet|Retreiver|Riddler|Rival IQ|scooter|Scrapy|Scrubby|searchsight|seekbot|semanticdiscovery|SemrushBot|Simpy|SimplePie|SEOstats|SimpleRSS|SiteCon|Slackbot-LinkExpanding|Slack-ImgProxy|Slurp|snappy|Speedy Spider|Squrl Java|Stringer|TheUsefulbot|ThumbShotsBot|Thumbshots\.ru|Tiny Tiny RSS|Twitterbot|WhatsApp|URL2PNG|Vagabondo|VoilaBot|^vortex|Votay bot|^voyager|WASALive.Bot|Web-sniffer|WebThumb|WeSEE:[A-z]{1,30}|WhatWeb|WIRE|WordPress|Wotbox|www\.almaden\.ibm\.com|Xenu(?:.s|) Link Sleuth|Xerka [A-z]{1,30}Bot|yacy(?:bot|)|YahooSeeker|Yahoo! Slurp|Yandex\w{1,30}|YodaoBot(?:-[A-z]{1,30}|)|YottaaMonitor|Yowedo|^Zao|^Zao-Crawler|ZeBot_www\.ze\.bz|ZooShot|ZyBorg|ArcGIS Hub Indexer|GPTBot)(?:[ /]v?(\d+)(?:\.(\d+)(?:\.(\d+)|)|)|)'
 
   # AWS S3 Clients
   # must come before "Bots General matcher" to catch "boto"/"boto3" before "bot"
-  - regex: '\b(Boto3?|JetS3t|aws-(?:cli|sdk-(?:cpp|go|java|nodejs|ruby2?|dotnet-(?:\d{1,2}|core)))|s3fs)/(\d+)\.(\d+)(?:\.(\d+)|)'
+  - regex: '\b(Boto3?|JetS3t|aws-(?:cli|sdk-(?:cpp|go|go-v\d|java|nodejs|ruby2?|dotnet-(?:\d{1,2}|core)))|s3fs)/(\d+)\.(\d+)(?:\.(\d+)|)'
 
   # SAFE FME
   - regex: '(FME)\/(\d+\.\d+)\.(\d+)\.(\d+)'
@@ -179,6 +179,9 @@ user_agent_parsers:
   - regex: '\[FB.{0,300};'
     family_replacement: 'Facebook'
 
+  # RecipeRadar crawler
+  - regex: '(RecipeRadar)/(\d+)\.(\d+)(?:\.(\d+)|)'
+
   # Bots General matcher 'name/0.0'
   - regex: '^.{0,200}?(?:\/[A-Za-z0-9\.]{0,50}|) {0,2}([A-Za-z0-9 \-_\!\[\]:]{0,50}(?:[Aa]rchiver|[Ii]ndexer|[Ss]craper|[Bb]ot|[Ss]pider|[Cc]rawl[a-z]{0,50}))[/ ](\d+)(?:\.(\d+)(?:\.(\d+)|)|)'
   # Bots containing bot(but not CUBOT)
@@ -215,6 +218,16 @@ user_agent_parsers:
   # Twitter
   - regex: '(Twitter for (?:iPhone|iPad)|TwitterAndroid)(?:\/(\d+)\.(\d+)|)'
     family_replacement: 'Twitter'
+  # TikTok
+  - regex: '(musical_ly) app_version\/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'TikTok'
+  - regex: '(musical_ly_)(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'TikTok'
+  - regex: '(BytedanceWebview)\/[a-z0-9]+'
+    family_replacement: 'TikTok'
+  # KakaoTalk
+  - regex: 'Mozilla.{1,200}Mobile.{1,100}(KAKAOTALK)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'KakaoTalk'
 
   # Phantom app
   - regex: 'Mozilla.{1,200}Mobile.{1,100}(Phantom\/ios|Phantom\/android).(\d+)\.(\d+)\.(\d+)'
@@ -394,7 +407,7 @@ user_agent_parsers:
   - regex: '(Instabridge)/(\d+)(?:\.(\d+)|)(?:\.(\d+)|)'
 
   # Aloha Browser
-  - regex: '(AlohaBrowser)/(\d+)\.(\d+)\.(\d+)(?:\.(\d+)|)'
+  - regex: '(AlohaBrowser|ABB)/(\d+)\.(\d+)\.(\d+)(?:\.(\d+)|)'
     family_replacement: 'Aloha Browser'
 
   # Brave Browser https://brave.com/ , should go before Safari and Chrome Mobile
@@ -415,7 +428,7 @@ user_agent_parsers:
     family_replacement: 'Edge Mobile'
 
   # Oculus Browser, should go before Samsung Internet
-  - regex: '(OculusBrowser)/(\d+)\.(\d+).0.0(?:\.([0-9\-]+)|)'
+  - regex: '(OculusBrowser)/(\d+)\.(\d+)(?:\.([0-9\-]+)|)'
     family_replacement: 'Oculus Browser'
 
   # Samsung Internet (based on Chrome, but lacking some features)
@@ -492,6 +505,12 @@ user_agent_parsers:
   - regex: '(Ecosia) android@(\d+)(?:\.(\d+)|)(?:\.(\d+)|)(?:\.(\d+)|)'
     family_replacement: 'Ecosia Android'
 
+  # VivoBrowser
+  - regex: '(VivoBrowser)\/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+
+  # HiBrowser
+  - regex: '(HiBrowser)\/v(\d+)\.(\d+)\.(\d+)\.(\d+)'
+
   # Chrome Mobile
   - regex: 'Version/.{1,300}(Chrome)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
     family_replacement: 'Chrome Mobile WebView'
@@ -569,6 +588,85 @@ user_agent_parsers:
   - regex: '^(surveyon)/(\d+)\.(\d+)\.(\d+)'
     family_replacement: 'Surveyon'
 
+  # 115 Browser
+  - regex: '(115Browser)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: '115 Browser'
+
+ # Avira
+  - regex: '(Avira)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Avira'
+
+  # CCleaner Browser
+  - regex: '(CCleaner)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'CCleaner'
+
+  # Norton
+  - regex: '(Norton)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Norton'
+
+  # Quark
+  - regex: '(Quark)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Quark'
+  # Quark PC
+  - regex: '(QuarkPC)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Quark PC'
+
+  # Smart Lenovo Browser
+  - regex: '(SLBrowser)/(\d+)\.(\d+)\.(\d+)\.(\d+) SLBChan/(\d+)'
+    family_replacement: 'Smart Lenovo Browser'
+
+  # Atom Browser
+  - regex: '(Atom)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Atom Browser'
+
+  # 360 Secure Browser
+  - regex: '(Chrome)/\d+\.\d+\.\d+\.\d+ .* QIHU 360(?:SEi18n|ENT)'
+    family_replacement: '360 Secure Browser'
+
+  # Decentr Web3 Browser
+  - regex: '(Decentr)'
+    family_replacement: 'Decentr Web3 Browser'
+
+  # Sparrow Browser
+  - regex: '(Sparrow)'
+    family_replacement: 'Sparrow Browser'
+
+  # Chromium GOST Browser
+  - regex: '(Chromium GOST)'
+    family_replacement: 'Chromium GOST Browser'
+
+  # AOL Shield Browser
+  - regex: '(AOLShield)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'AOL Shield Browser'
+
+  # Hola Browser
+  - regex: '(Hola)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Hola Browser'
+
+  # Craving Explorer Browser
+  - regex: '(CravingExplorer)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Craving Explorer Browser'
+
+  # Talon Cyber Security Browser
+  - regex: '(Talon)'
+    family_replacement: 'Talon Cyber Security Browser'
+
+  # QAX Browser
+  - regex: '(Qaxbrowser)'
+    family_replacement: 'QAX Browser'
+
+  # AOL Desktop Gold Browser
+  - regex: '(ADG)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'AOL Desktop Gold Browser'
+
+  # Sber Browser
+  - regex: '(SberBrowser)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Sber Browser'
+
+  # JiSu Browser
+  - regex: '(JiSu)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'JiSu Browser'
+
   #### END SPECIAL CASES TOP ####
 
   #### MAIN CASES - this catches > 50% of all browsers ####
@@ -1388,6 +1486,13 @@ os_parsers:
   - regex: '^Box.{0,200};(Darwin)/(10)\.(1\d)(?:\.(\d+)|)'
     os_replacement: 'Mac OS X'
 
+  ##########
+  # Hashicorp API
+  # APN/1.0 HashiCorp/1.0 Terraform/1.8.0 (+https://www.terraform.io) terraform-provider-aws/4.67.0 (+https://registry.terraform.io/providers/hashicorp/aws) aws-sdk-go/1.44.261 (go1.19.8; darwin; arm64)
+  ##########
+  - regex: 'darwin; arm64'
+    os_replacement: 'Mac OS X'
+
   ##########
   # iOS
   # http://en.wikipedia.org/wiki/IOS_version_history
@@ -1672,29 +1777,27 @@ os_parsers:
   - regex: 'CFNetwork/.{0,100} Darwin/(21)\.\d+'
     os_replacement: 'iOS'
     os_v1_replacement: '15'
-  - regex: 'CFNetwork/.{0,100} Darwin/22\.0\.\d+'
+  - regex: 'CFNetwork/.{0,100} Darwin/22\.([0-5])\.\d+'
     os_replacement: 'iOS'
     os_v1_replacement: '16'
-    os_v2_replacement: '0'
-  - regex: 'CFNetwork/.{0,100} Darwin/22\.1\.\d+'
-    os_replacement: 'iOS'
-    os_v1_replacement: '16'
-    os_v2_replacement: '1'
-  - regex: 'CFNetwork/.{0,100} Darwin/22\.2\.\d+'
-    os_replacement: 'iOS'
-    os_v1_replacement: '16'
-    os_v2_replacement: '2'
-  - regex: 'CFNetwork/.{0,100} Darwin/22\.3\.\d+'
-    os_replacement: 'iOS'
-    os_v1_replacement: '16'
-    os_v2_replacement: '3'
-  - regex: 'CFNetwork/.{0,100} Darwin/22\.4\.\d+'
-    os_replacement: 'iOS'
-    os_v1_replacement: '16'
-    os_v2_replacement: '4'
+    os_v2_replacement: '$1'
   - regex: 'CFNetwork/.{0,100} Darwin/(22)\.\d+'
     os_replacement: 'iOS'
     os_v1_replacement: '16'
+  - regex: 'CFNetwork/.{0,100} Darwin/23\.([0-5])\.\d+'
+    os_replacement: 'iOS'
+    os_v1_replacement: '17'
+    os_v2_replacement: '$1'
+  - regex: 'CFNetwork/.{0,100} Darwin/(23)\.\d+'
+    os_replacement: 'iOS'
+    os_v1_replacement: '17'
+  - regex: 'CFNetwork/.{0,100} Darwin/24\.([0-5])\.\d+'
+    os_replacement: 'iOS'
+    os_v1_replacement: '18'
+    os_v2_replacement: '$1'
+  - regex: 'CFNetwork/.{0,100} Darwin/(24)\.\d+'
+    os_replacement: 'iOS'
+    os_v1_replacement: '18'
   - regex: 'CFNetwork/.{0,100} Darwin/'
     os_replacement: 'iOS'
 
@@ -1889,6 +1992,21 @@ os_parsers:
   # Roku Digital-Video-Players https://www.roku.com/
   - regex: '^(Roku)/DVP-(\d+)\.(\d+)'
 
+  ##########
+  # Amazon S3 client boto3
+  # Hasicorp API
+  # Boto3/1.28.62 md/Botocore#1.31.62 ua/2.0 os/macos#22.4.0 md/arch#arm64 lang/python#3.11.6 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.62
+  # APN/1.0 HashiCorp/1.0 Terraform/1.8.1 (+https://www.terraform.io) terraform-provider-aws/4.67.0 (+https://registry.terraform.io/providers/hashicorp/aws) aws-sdk-go-v2/1.18.0 os/macos lang/go/1.19.8 md/GOOS/darwin md/GOARCH/arm64 api/identitystore/1.16.11
+  ##########
+  - regex: 'os\/macos[#]?(\d*)[.]?(\d*)[.]?(\d*)'
+    os_replacement: 'Mac OS X'
+    os_v1_replacement: '$1'
+    os_v2_replacement: '$2'
+    os_v3_replacement: '$3'
+
+  # Huawei HarmonyOS
+  - regex: '(HarmonyOS)[\s;]+(\d+|)\.?(\d+|)\.?(\d+|)'
+
 device_parsers:
 
   #########
@@ -3097,7 +3215,7 @@ device_parsers:
     device_replacement: 'HTC $1'
     brand_replacement: 'HTC'
     model_replacement: '$1'
-  - regex: '; {0,2}(ADR6200|ADR6400L|ADR6425LVW|Amaze|DesireS?|EndeavorU|Eris|EVO|Evo\d[A-Z]+|HD2|IncredibleS?|Inspire[A-Z0-9]*|Inspire[A-Z0-9]*|Sensation[A-Z0-9]*|Wildfire)[ _-](.{1,200}?)(?:[/;\)]|Build|MIUI|1\.0)'
+  - regex: '; {0,2}(ADR6200|ADR6400L|ADR6425LVW|Amaze|DesireS?|EndeavorU|Eris|EVO|Evo\d[A-Z]+|HD2|IncredibleS?|Inspire[A-Z0-9]*|Sensation[A-Z0-9]*|Wildfire)[ _-](.{1,200}?)(?:[/;\)]|Build|MIUI|1\.0)'
     regex_flag: 'i'
     device_replacement: 'HTC $1 $2'
     brand_replacement: 'HTC'
@@ -5905,7 +6023,7 @@ device_parsers:
   ##########
   # Spiders (this is a hack...)
   ##########
-  - regex: '^.{0,100}(bot|BUbiNG|zao|borg|DBot|oegp|silk|Xenu|zeal|^NING|CCBot|crawl|htdig|lycos|slurp|teoma|voila|yahoo|Sogou|CiBra|Nutch|^Java/|^JNLP/|Daumoa|Daum|Genieo|ichiro|larbin|pompos|Scrapy|snappy|speedy|spider|msnbot|msrbot|vortex|^vortex|crawler|favicon|indexer|Riddler|scooter|scraper|scrubby|WhatWeb|WinHTTP|bingbot|BingPreview|openbot|gigabot|furlbot|polybot|seekbot|^voyager|archiver|Icarus6j|mogimogi|Netvibes|blitzbot|altavista|charlotte|findlinks|Retreiver|TLSProber|WordPress|SeznamBot|ProoXiBot|wsr\-agent|Squrl Java|EtaoSpider|PaperLiBot|SputnikBot|A6\-Indexer|netresearch|searchsight|baiduspider|YisouSpider|ICC\-Crawler|http%20client|Python-urllib|dataparksearch|converacrawler|Screaming Frog|AppEngine-Google|YahooCacheSystem|fast\-webcrawler|Sogou Pic Spider|semanticdiscovery|Innovazion Crawler|facebookexternalhit|Google.{0,200}/\+/web/snippet|Google-HTTP-Java-Client|BlogBridge|IlTrovatore-Setaccio|InternetArchive|GomezAgent|WebThumbnail|heritrix|NewsGator|PagePeeker|Reaper|ZooShot|holmes|NL-Crawler|Pingdom|StatusCake|WhatsApp|masscan|Google Web Preview|Qwantify|Yeti|OgScrper)'
+  - regex: '^.{0,100}(bot|BUbiNG|zao|borg|DBot|oegp|silk|Xenu|zeal|^NING|CCBot|crawl|htdig|lycos|slurp|teoma|voila|yahoo|Sogou|CiBra|Nutch|^Java/|^JNLP/|Daumoa|Daum|Genieo|ichiro|larbin|pompos|Scrapy|snappy|speedy|spider|msnbot|msrbot|vortex|^vortex|crawler|favicon|indexer|Riddler|scooter|scraper|scrubby|WhatWeb|WinHTTP|bingbot|BingPreview|openbot|gigabot|furlbot|polybot|seekbot|^voyager|archiver|Icarus6j|mogimogi|Netvibes|blitzbot|altavista|charlotte|findlinks|Retreiver|TLSProber|WordPress|SeznamBot|ProoXiBot|wsr\-agent|Squrl Java|EtaoSpider|PaperLiBot|SputnikBot|A6\-Indexer|netresearch|searchsight|baiduspider|YisouSpider|ICC\-Crawler|http%20client|Python-urllib|dataparksearch|converacrawler|Screaming Frog|AppEngine-Google|YahooCacheSystem|fast\-webcrawler|Sogou Pic Spider|semanticdiscovery|Innovazion Crawler|facebookexternalhit|Google.{0,200}/\+/web/snippet|Google-HTTP-Java-Client|BlogBridge|IlTrovatore-Setaccio|InternetArchive|GomezAgent|WebThumbnail|heritrix|NewsGator|PagePeeker|Reaper|ZooShot|holmes|NL-Crawler|Pingdom|StatusCake|WhatsApp|masscan|Google Web Preview|Qwantify|Yeti|OgScrper|RecipeRadar|GPTBot)'
     regex_flag: 'i'
     device_replacement: 'Spider'
     brand_replacement: 'Spider'

db/migrations/20240711224103_create_clicks.sql

@@ -4,7 +4,6 @@ CREATE TABLE clicks (
 	id TEXT PRIMARY KEY NOT NULL,
 	link_id TEXT NOT NULL,
 	user_agent TEXT,
-	language TEXT,
 	browser TEXT,
 	os TEXT,
 	source TEXT,

db/migrations/20250316102350_add_country_to_clicks.sql

@@ -0,0 +1,8 @@
+-- +micrate Up
+-- SQL in section 'Up' is executed when this migration is applied
+ALTER TABLE clicks ADD COLUMN country TEXT;
+ALTER TABLE clicks RENAME COLUMN source TO referer;
+
+-- +micrate Down
+-- SQL section 'Down' is executed when this migration is rolled back
+ALTER TABLE clicks RENAME COLUMN referer TO source;

db/migrations/20250316111734_replace_unkwown_with_null.sql

@@ -0,0 +1,13 @@
+-- +micrate Up
+-- SQL in section 'Up' is executed when this migration is applied
+UPDATE clicks SET user_agent = NULL WHERE user_agent = 'Unknown';
+UPDATE clicks SET browser = NULL WHERE browser = 'Unknown';
+UPDATE clicks SET os = NULL WHERE os = 'Unknown';
+UPDATE clicks SET referer = NULL WHERE referer = 'Unknown';
+
+-- +micrate Down
+-- SQL section 'Down' is executed when this migration is rolled back
+UPDATE clicks SET user_agent = 'Unknown' WHERE user_agent IS NULL;
+UPDATE clicks SET browser = 'Unknown' WHERE browser IS NULL;
+UPDATE clicks SET os = 'Unknown' WHERE os IS NULL;
+UPDATE clicks SET referer = 'Unknown' WHERE referer IS NULL;

db/migrations/20250318072657_replace_slug_index_with_covering_index.sql

@@ -0,0 +1,9 @@
+-- +micrate Up
+-- SQL in section 'Up' is executed when this migration is applied
+DROP INDEX IF EXISTS idx_links_slug;      -- Remove old composite index
+CREATE INDEX IF NOT EXISTS idx_links_slug_optimized ON links (slug, url);
+
+-- +micrate Down
+-- SQL in section 'Down' is executed when this migration is rolled back
+DROP INDEX IF EXISTS idx_links_slug_optimized;
+CREATE INDEX IF NOT EXISTS idx_links_slug ON links (id, slug, url);

db/migrations/20250319192003_convert_all_tables_text_ids_to_integer.sql

@@ -0,0 +1,102 @@
+-- +micrate Up
+-- SQL in section 'Up' is executed when this migration is applied
+-- 1. Create new users table with INTEGER PK
+CREATE TABLE users_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    name VARCHAR(100) NOT NULL,
+    api_key VARCHAR(64) UNIQUE NOT NULL,
+    created_at INTEGER DEFAULT CURRENT_TIMESTAMP NOT NULL,
+    updated_at INTEGER DEFAULT CURRENT_TIMESTAMP NOT NULL
+);
+
+-- Create a mapping table to track old and new user IDs
+CREATE TEMPORARY TABLE user_id_map (
+    old_id TEXT,
+    new_id INTEGER
+);
+
+-- Insert users data and capture the mappings
+INSERT INTO users_new (name, api_key, created_at, updated_at)
+SELECT name, api_key, created_at, updated_at FROM users;
+
+INSERT INTO user_id_map
+SELECT u.id, u_new.id
+FROM users u
+JOIN users_new u_new ON u_new.api_key = u.api_key;
+
+-- 2. Create new links table with INTEGER PK
+CREATE TABLE links_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    user_id INTEGER NOT NULL,
+    slug VARCHAR(8) UNIQUE NOT NULL,
+    url TEXT NOT NULL,
+    created_at INTEGER DEFAULT CURRENT_TIMESTAMP NOT NULL,
+    updated_at INTEGER DEFAULT CURRENT_TIMESTAMP NOT NULL,
+    FOREIGN KEY (user_id) REFERENCES users_new(id) ON DELETE CASCADE
+);
+
+-- Create a mapping table for links
+CREATE TEMPORARY TABLE link_id_map (
+    old_id TEXT,
+    new_id INTEGER
+);
+
+-- Insert links data with new user_id foreign keys
+INSERT INTO links_new (user_id, slug, url, created_at, updated_at)
+SELECT 
+    (SELECT new_id FROM user_id_map WHERE old_id = l.user_id),
+    l.slug,
+    l.url,
+    l.created_at,
+    l.updated_at
+FROM links l;
+
+-- Create the mapping for links
+INSERT INTO link_id_map
+SELECT l.id, l_new.id
+FROM links l
+JOIN links_new l_new ON l_new.slug = l.slug AND l_new.url = l.url;
+
+-- 3. Create new clicks table with INTEGER PK
+CREATE TABLE clicks_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    link_id INTEGER NOT NULL,
+    user_agent TEXT,
+    browser TEXT,
+    os TEXT,
+    referer TEXT,
+    country TEXT,
+    created_at INTEGER DEFAULT CURRENT_TIMESTAMP NOT NULL,
+    updated_at INTEGER DEFAULT CURRENT_TIMESTAMP NOT NULL,
+    FOREIGN KEY (link_id) REFERENCES links_new(id) ON DELETE CASCADE
+);
+
+-- Insert clicks data with new link_id foreign keys
+INSERT INTO clicks_new (link_id, user_agent, browser, os, referer, country, created_at, updated_at)
+SELECT 
+    (SELECT new_id FROM link_id_map WHERE old_id = c.link_id),
+    c.user_agent,
+    c.browser,
+    c.os,
+    c.referer,
+    c.country,
+    c.created_at,
+    c.updated_at
+FROM clicks c;
+
+-- 4. Drop old tables and rename new tables
+DROP TABLE clicks;
+DROP TABLE links;
+DROP TABLE users;
+
+ALTER TABLE clicks_new RENAME TO clicks;
+ALTER TABLE links_new RENAME TO links;
+ALTER TABLE users_new RENAME TO users;
+
+-- 5. Drop unused indexes
+DROP INDEX IF EXISTS index_users_api_key;
+DROP INDEX IF EXISTS idx_links_slug;
+DROP INDEX IF EXISTS idx_links_slug_optimized;
+
+-- +micrate Down
+-- SQL section 'Down' is executed when this migration is rolled back

db/seed.sql

@@ -0,0 +1,69 @@
+INSERT INTO users (name, api_key)
+VALUES
+('User 1', 'secure_api_key_1'),
+('User 2', 'secure_api_key_2');
+
+-- Create 10,000 links (5,000 per user)
+WITH RECURSIVE link_numbers(n) AS (
+    SELECT 1
+    UNION ALL
+    SELECT n+1 FROM link_numbers
+    LIMIT 10000
+)
+INSERT INTO links (user_id, slug, url)
+SELECT
+    ((n-1) % 2) + 1, -- User ID (1-2)
+    'slug' || n,      -- Unique slug
+    'https://sjdonado.com/page/' || n
+FROM link_numbers;
+
+-- Create 1,000 clicks per link (10 million total)
+WITH RECURSIVE counts(n) AS (
+    SELECT 1
+    UNION ALL
+    SELECT n+1 FROM counts
+    LIMIT 1000
+)
+INSERT INTO clicks (link_id, user_agent, browser, os, referer, country)
+SELECT
+    l.id,
+    CASE (c.n % 5)
+        WHEN 0 THEN 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'
+        WHEN 1 THEN 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15)'
+        WHEN 2 THEN 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_0)'
+        WHEN 3 THEN 'Mozilla/5.0 (X11; Linux x86_64)'
+        ELSE 'Mozilla/5.0 (Android 11; Mobile)'
+    END,
+    CASE (c.n % 3)
+        WHEN 0 THEN 'Firefox'
+        WHEN 1 THEN 'Chrome'
+        ELSE 'Safari'
+    END,
+    CASE (c.n % 4)
+        WHEN 0 THEN 'macOS'
+        WHEN 1 THEN 'Windows'
+        WHEN 2 THEN 'iOS'
+        ELSE 'Android'
+    END,
+    CASE (c.n % 6)
+        WHEN 0 THEN 'https://sjdonado.com'
+        WHEN 1 THEN 'https://donado.co'
+        WHEN 2 THEN 'https://idonthavespotify.donado.co'
+        WHEN 3 THEN 'https://spookyplanning.com'
+        WHEN 4 THEN 'https://github.com/sjdonado'
+        ELSE NULL
+    END,
+    CASE (c.n % 10)
+        WHEN 0 THEN 'Colombia'
+        WHEN 1 THEN 'Brazil'
+        WHEN 2 THEN 'Canada'
+        WHEN 3 THEN 'Germany'
+        WHEN 4 THEN 'France'
+        WHEN 5 THEN 'Japan'
+        WHEN 6 THEN 'Australia'
+        WHEN 7 THEN 'Brazil'
+        WHEN 8 THEN 'India'
+        ELSE 'China'
+    END
+FROM links l
+CROSS JOIN counts c;

docs/API.md

@@ -4,45 +4,27 @@
 
    - Endpoint: `GET /api/ping`
    - Payload: None
+   - Response: 200
    - Response Example
      ```json
      {
-       "message": "pong"
+       "data": "pong"
      }
      ```
-
 2. **Redirect by Slug**
 
    - Endpoint: `GET /:slug`
-   - Headers: `X-Api-Key`
    - Payload: None
-   - Response Example
-     ```json
-     {
-       "data": {
-         "id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db",
-         "refer": "http://localhost:4000/3wP4BQ",
-         "origin": "https://monocuco.donado.co",
-         "clicks": [
-           {
-             "id": "730e2202-58f9-478c-a24c-f1c561df6716",
-             "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0",
-             "language": "en-US",
-             "browser": "Firefox",
-             "os": "Mac OS X",
-             "source": "Unknown",
-             "created_at": "2024-07-12T19:25:22Z"
-           }
-         ]
-       }
-     }
-     ```
+   - Response: 301
 
 3. **List All Links**
 
    - Endpoint: `GET /api/links`
    - Headers: `X-Api-Key`
-   - Payload: None
+   - Query Parameters:
+     - `limit` (optional): Number of results per page (default: 100)
+     - `cursor` (optional): Pagination cursor from previous response
+   - Response: 200
    - Response Example
      ```json
      {
@@ -50,28 +32,22 @@
          {
            "id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db",
            "refer": "http://localhost:4000/3wP4BQ",
-           "origin": "https://monocuco.donado.co",
-           "clicks": [
-             {
-               "id": "730e2202-58f9-478c-a24c-f1c561df6716",
-               "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0",
-               "language": "en-US",
-               "browser": "Firefox",
-               "os": "Mac OS X",
-               "source": "Unknown",
-               "created_at": "2024-07-12T19:25:22Z"
+           "origin": "https://monocuco.donado.co"
          }
-           ]
+       ],
+       "pagination": {
+         "has_more": true,
+         "next": "75e0a7f4-9c5e-1235-b546-eb9c5e40f7ac"
        }
-       ]
      }
      ```
 
 4. **List link by ID**
-
    - Endpoint: `GET /api/links/:id`
    - Headers: `X-Api-Key`
    - Payload: None
+   - Note: This endpoint returns up to 100 of the most recent clicks. For complete click history, use the `/api/links/:id/clicks` endpoint with pagination.
+   - Response: 200
    - Response Example
      ```json
      {
@@ -83,10 +59,10 @@
            {
              "id": "730e2202-58f9-478c-a24c-f1c561df6716",
              "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0",
-             "language": "en-US",
+             "country": "DE",
              "browser": "Firefox",
              "os": "Mac OS X",
-             "source": "Unknown",
+             "referer": "Direct",
              "created_at": "2024-07-12T19:25:22Z"
            }
          ]
@@ -94,9 +70,36 @@
      }
      ```
 
-5. **Create new link**
+5. **List Clicks for a Link**
+   - Endpoint: `GET /api/links/:id/clicks`
+   - Headers: `X-Api-Key`
+   - Query Parameters:
+     - `limit` (optional): Number of results per page (default: 100)
+     - `cursor` (optional): Pagination cursor from previous response
+   - Response: 200
+   - Response Example
+     ```json
+     {
+       "data": [
+         {
+           "id": "730e2202-58f9-478c-a24c-f1c561df6716",
+           "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0",
+           "country": "DE",
+           "browser": "Firefox",
+           "os": "Mac OS X",
+           "referer": "Direct",
+           "created_at": "2024-07-12T19:25:22Z"
+         }
+       ],
+       "pagination": {
+         "has_more": true,
+         "next": "629e3301-47f8-389b-b24c-f1c561df9825"
+       }
+     }
+     ```
 
-   - Endpoint\*\*: `POST /api/links`
+6. **Create new link**
+   - Endpoint: `POST /api/links`
    - Payload:
      ```json
      {
@@ -104,20 +107,20 @@
      }
      ```
    - Headers: `X-Api-Key`
+   - Response: 201
    - Response Example:
      ```json
      {
        "data": {
          "id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db",
          "refer": "http://localhost:4000/3wP4BQ",
-         "origin": "https://monocuco.donado.co/test",
+         "origin": "https://example.com",
          "clicks": []
        }
      }
      ```
 
-6. **Update an existing link by ID**
-
+7. **Update an existing link by ID**
    - Endpoint: `PUT /api/links/:id`
    - Payload:
      ```json
@@ -126,6 +129,7 @@
      }
      ```
    - Headers: `X-Api-Key`
+   - Response: 200
    - Response Example:
      ```json
      {
@@ -138,15 +142,8 @@
      }
      ```
 
-7. **Delete a link by ID**
-
+8. **Delete a link by ID**
   - Endpoint: `DELETE /api/links/:id`
   - Payload: None
   - Headers: `X-Api-Key`
-   - Response Example:
-     ```json
-     {
-       "message": "Link deleted"
-     }
-     ```
-
+  - Response: 204

docs/SETUP.md

@@ -6,6 +6,7 @@ Options:
   --create-user=NAME     Create a new user with the given name
   --list-users           List all users
   --delete-user=USER_ID  Delete a user by ID
+  --update-parsers       Download all required data files
 ```
 
 ## Run It Anywhere
@@ -26,17 +27,17 @@ docker run \
     --name bit \
     -p 4000:4000 \
     -e ENV="production" \
-    -e DATABASE_URL="sqlite3://./sqlite/data.db?journal_mode=wal&synchronous=normal&foreign_keys=true" \
+    -e DATABASE_URL="sqlite3://./sqlite/data.db" \
     -e APP_URL="http://localhost:4000" \
     -e ADMIN_NAME="Admin" \
     -e ADMIN_API_KEY=$(openssl rand -base64 32) \
     sjdonado/bit
 
-# Optional: Generate an api key
+# Create a new user
 # docker exec -it bit cli --create-user=Admin
 ```
 
-### Self-Hosted with Dokku
+### Dokku
 
 ```dockerfile
 FROM sjdonado/bit
@@ -51,15 +52,24 @@ dokku letsencrypt:enable bit
 dokku storage:ensure-directory bit-sqlite
 dokku storage:mount bit /var/lib/dokku/data/storage/bit-sqlite:/usr/src/app/sqlite/
 
-dokku config:set bit DATABASE_URL="sqlite3://./sqlite/data.db?journal_mode=wal&synchronous=normal&foreign_keys=true" APP_URL=https://bit.donado.co ADMIN_NAME=Admin ADMIN_API_KEY=$(openssl rand -base64 32)
+dokku config:set bit DATABASE_URL="sqlite3://./sqlite/data.db" APP_URL=https://bit.donado.co ADMIN_NAME=Admin ADMIN_API_KEY=$(openssl rand -base64 32)
 
 dokku ports:add bit http:80:4000
 dokku ports:add bit https:443:4000
 
-# Optional: Generate an api key
+# Create a new user
 # dokku run bit cli --create-user=Admin
 ```
 
+### Dokku (same network)
+Recommended for lower latency communication (no host network traversal)
+
+```bash
+  dokku network:create bit-net
+  dokku network:set bit attach-post-create bit-net
+  dokku network:set myapp attach-post-create bit-net
+```
+
 ## Local Development
 
 ### Requirements
@@ -99,49 +109,59 @@ ENV=test crystal spec
 
 ## Benchmark
 
-Conducted on a MacBook Air M2 with 16GB RAM.
+- Colima: cpu 1, mem 1
+- SoC: Apple M3 Pro
 
 ```
-$ ./benchmark.sh
+~/p/bit> colima start --cpu 1 --memory 1
+INFO[0000] starting colima
+INFO[0000] runtime: docker
+INFO[0001] starting ...                                  context=vm
+INFO[0076] provisioning ...                              context=docker
+INFO[0077] starting ...                                  context=docker
+INFO[0077] done
+~/p/bit> ./benchmark.cr
 Setting up...
-[+] Running 3/3
- ✔ Network bit_default       Created                                                                         0.0s
- ✔ Volume "bit_sqlite_data"  Created                                                                         0.0s
- ✔ Container bit             Started                                                                         0.1s
-Captured API Key: aHOCnZSuo2kOHy2mDa-iOA
 Waiting for the application to be ready...
-HTTP/1.1 200 OK
-Connection: keep-alive
-Content-Type: application/json
-Date: Sun, 27 Oct 2024 11:52:33 GMT
-Access-Control-Allow-Origin: *
-Access-Control-Allow-Methods: GET, POST, PUT, DELETE, OPTIONS
-Access-Control-Allow-Headers: Content-Type, Accept, Origin, X-Api-Key
-Content-Length: 13
-
-Starting resource usage monitoring...
-Creating 10000 short links with 100 conrurrent requests...
-Link creation complete: 10000 links created.
+Seeding the database...
+Checking seed results...
 Fetching all created links from /api/links...
-Selected link for benchmarking: http://localhost:4000/UaVZjA
+Selected link for benchmarking: http://localhost:4000/slug2576
 Starting benchmark with Bombardier...
-Bombarding http://localhost:4000/oEKLAg with 10000 request(s) using 100 connection(s)
- 10000 / 10000 [===============================================================================] 100.00% 830/s 12s
+Bombarding http://localhost:4000/slug2576 with 100000 request(s) using 125 connection(s)
+ 100000 / 100000 [===================================================================================================================================================================] 100.00% 7795/s 12s
 Done!
 Statistics        Avg      Stdev        Max
-  Reqs/sec       853.89    1625.49    8942.54
-  Latency      118.48ms    11.52ms   142.58ms
+  Reqs/sec      7900.70    7570.15   29263.59
+  Latency       15.89ms    10.22ms    67.32ms
+  Latency Distribution
+     50%     4.82ms
+     75%     9.24ms
+     90%    51.61ms
+     95%    52.74ms
+     99%    55.07ms
   HTTP codes:
-    1xx - 0, 2xx - 0, 3xx - 10000, 4xx - 0, 5xx - 0
+    1xx - 0, 2xx - 0, 3xx - 100000, 4xx - 0, 5xx - 0
     others - 0
-  Throughput:   360.02KB/s
-Benchmark completed.
+  Throughput:     2.14MB/s
+Benchmark completed successfully.
 Analyzing resource usage...
-**** Results ****
-Average CPU Usage: 40.68%
-Average Memory Usage: 28.62 MiB
-./benchmark.sh: line 135: 61567 Terminated: 15          monitor_resource_usage
-[+] Running 2/2
- ✔ Container bit        Removed                                                                                                                                                                                               10.1s
- ✔ Network bit_default  Removed  
+Timestamp       CPU(%)  Memory(MiB)
+1742763202      0.01    44.83
+1742763204      0.01    44.78
+1742763206      91.53   68.23
+1742763208      92.03   68.17
+1742763210      91.0    68.09
+1742763212      92.73   68.38
+1742763214      92.17   67.66
+1742763216      91.1    67.69
+1742763218      2.93    67.04
+
+**** Resource Usage Statistics ****
+  Measurements: 9
+  Average CPU Usage: 61.5%
+  Average Memory Usage: 62.76 MiB
+  Peak CPU Usage: 92.73%
+  Peak Memory Usage: 68.38 MiB
+Cleanup completed. Resource usage data saved in resource_usage.txt
 ```

download_regexes.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-REGEXES_URL="https://raw.githubusercontent.com/ua-parser/uap-core/master/regexes.yaml"
-DOWNLOAD_DIR="data"
-REGEXES_FILE="regexes.yaml"
-
-mkdir -p $DOWNLOAD_DIR
-
-curl -L -o $DOWNLOAD_DIR/$REGEXES_FILE $REGEXES_URL
-
-echo "Regexes file downloaded to $DOWNLOAD_DIR/$REGEXES_FILE"

scripts/cli.cr

@@ -1,4 +1,3 @@
-require "uuid"
 require "option_parser"
 
 require "../app/services/cli"
@@ -19,11 +18,20 @@ OptionParser.parse do |parser|
     exit
   end
 
+  parser.on("--update-parsers", "Download UA regexes and/or GeoLite2 database") do
+    puts "=== Starting data files update ==="
+    App::Services::Cli.update_uap_regexes
+    App::Services::Cli.download_geolite_db
+    puts "=== Data files updated successfully ==="
+    exit
+  end
+
   if ARGV.empty?
     puts "Usage: ./cli [options]"
     puts "Options:"
     puts "  --create-user=NAME     Create a new user with the given name"
     puts "  --list-users           List all users"
     puts "  --delete-user=USER_ID  Delete a user by ID"
+    puts "  --update-parsers          Download all required data files"
   end
 end

shard.lock

@@ -20,10 +20,18 @@ shards:
     git: https://github.com/crystal-loot/exception_page.git
     version: 0.4.1
 
+  ipaddress:
+    git: https://github.com/sija/ipaddress.cr.git
+    version: 0.2.3
+
   kemal:
     git: https://github.com/kemalcr/kemal.git
     version: 1.5.0
 
+  maxminddb:
+    git: https://github.com/delef/maxminddb.cr.git
+    version: 1.5.0
+
   micrate:
     git: https://github.com/amberframework/micrate.git
     version: 0.15.1
@@ -48,7 +56,3 @@ shards:
     git: https://github.com/crystal-lang/crystal-sqlite3.git
     version: 0.19.0
 
-  user_agent_parser:
-    git: https://github.com/busyloop/user_agent_parser.git
-    version: 2.0.1
-

shard.yml

@@ -1,5 +1,5 @@
 name: bit
-version: 1.4.1
+version: 1.5.3
 
 authors:
   - Juan Rodriguez <sjdonado@icloud.com>
@@ -20,9 +20,8 @@ dependencies:
   micrate:
     github: amberframework/micrate
     version: 0.15.1
-  user_agent_parser:
-    github: busyloop/user_agent_parser
-    version: 2.0.1
+  maxminddb:
+    github: delef/maxminddb.cr
 
 development_dependencies:
   dotenv:

spec/integration/link_spec.cr

@@ -82,33 +82,71 @@ describe "App::Controllers::Link" do
   end
 
   describe "Index" do
-    it "should redirect to origin domain" do
+    it "should redirect to origin domain with forwarded headers" do
       link = "https://test.com"
       test_user = create_test_user()
 
       test_link = create_test_link(test_user, link)
       serialized_link = App::Serializers::Link.new(test_link)
 
-      get(serialized_link.refer, headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
+      user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0"
+
+      get(serialized_link.refer, headers: HTTP::Headers{
+        "X-Api-Key" => test_user.api_key.to_s,
+        "User-Agent" => user_agent
+      })
 
       response.headers["Location"].should eq(link)
+      response.headers["User-Agent"].should eq(user_agent)
+      response.headers.has_key?("X-Forwarded-For").should be_true
     end
 
-    it "should create a new click after redirect" do
+    it "should create a new click after redirect with proper information" do
       link = "https://sjdonado.com"
       test_user = create_test_user()
 
       test_link = create_test_link(test_user, link)
       serialized_link = App::Serializers::Link.new(test_link)
 
-      get(serialized_link.refer, headers: HTTP::Headers{"User-Agent" => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0"})
+      user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0"
+      referer = "https://example.com/page"
+
+      get(serialized_link.refer, headers: HTTP::Headers{
+        "User-Agent" => user_agent,
+        "Referer" => referer
+      })
 
       Fiber.yield # replace yield with sleep 5 to debug errors
 
       response.headers["Location"].should eq(link)
 
+      # Verify that the click was recorded
       updated_test_link = get_test_link(test_link.id)
       updated_test_link.clicks.size.should eq(test_link.clicks.size + 1)
+
+      # Verify click details
+      latest_click = updated_test_link.clicks.last
+      latest_click.user_agent.should eq(user_agent)
+      latest_click.browser.should eq("Firefox")
+      latest_click.os.should eq("Mac OS X")
+      latest_click.referer.should eq("example.com") # Should extract host from the referer
+    end
+
+    it "should create a click with utm_source when no referer is provided" do
+      link = "https://sjdonado.com"
+      test_user = create_test_user()
+
+      test_link = create_test_link(test_user, link)
+      serialized_link = App::Serializers::Link.new(test_link)
+
+      # Add utm_source parameter
+      get("#{serialized_link.refer}?utm_source=email_campaign")
+
+      Fiber.yield
+
+      updated_test_link = get_test_link(test_link.id)
+      latest_click = updated_test_link.clicks.last
+      latest_click.referer.should eq("email_campaign")
     end
 
     it "should return 404 - link does not exist" do
@@ -123,8 +161,8 @@ describe "App::Controllers::Link" do
   end
 
   describe "All" do
-    it "should return all links" do
-      links = ["https://google.com", "google.com", "google.com.co"]
+    it "should return all links with pagination" do
+      links = ["https://sjdonado.com", "sjdonado.com", "sjdonado.com.co"]
       test_user = create_test_user()
 
       links.each do |link|
@@ -133,14 +171,58 @@ describe "App::Controllers::Link" do
 
       get("/api/links", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
 
-      parsed_response = Hash(String, Array(Hash(String, String | Int64 | Array(Hash(String, String))))).from_json(response.body)
-      parsed_response["data"][0]["origin"].should eq(links[0])
-      parsed_response["data"][1]["origin"].should eq(links[1])
-      parsed_response["data"][2]["origin"].should eq(links[2])
+      parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body)
+
+      # Check that each link is in the response data
+      origins = parsed_response["data"].as(Array).map { |link| link["origin"] }
+      links.each do |link|
+        origins.should contain(link)
+      end
+
+      parsed_response["pagination"].as(Hash)["has_more"].should be_false
+    end
+
+    it "should respect custom limit parameter" do
+      test_user = create_test_user()
+
+      5.times do |i|
+        create_test_link(test_user, "https://example.com/#{i}")
+      end
+
+      get("/api/links?limit=2", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
+
+      parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body)
+      parsed_response["data"].as(Array).size.should eq(2)
+      parsed_response["pagination"].as(Hash)["has_more"].should be_true
+      parsed_response["pagination"].as(Hash)["next"].should_not be_nil
+    end
+
+    it "should support cursor-based pagination" do
+      test_user = create_test_user()
+
+      5.times do |i|
+        create_test_link(test_user, "https://example.com/#{i}")
+      end
+
+      # Get first page
+      get("/api/links?limit=2", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
+      first_page = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body)
+      cursor = first_page["pagination"].as(Hash)["next"]
+
+      # Get second page using cursor
+      get("/api/links?limit=2&cursor=#{cursor}", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
+      second_page = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body)
+
+      # Ensure different links are returned
+      first_page_ids = first_page["data"].as(Array).map { |link| link["id"] }
+      second_page_ids = second_page["data"].as(Array).map { |link| link["id"] }
+
+      # Check that no IDs from first page appear in second page
+      (first_page_ids & second_page_ids).empty?.should be_true
     end
 
     it "should return owned links only" do
-      links = ["https://google.de", "google.de", "google.edu.co", "x.com"]
+      links = ["https://donado.co", "donado.co", "uninorte.edu.co", "kagi.com"]
       test_user = create_test_user()
 
       links[0..2].each do |link|
@@ -152,11 +234,14 @@ describe "App::Controllers::Link" do
 
       get("/api/links", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
 
-      parsed_response = Hash(String, Array(Hash(String, String | Int64 | Array(Hash(String, String))))).from_json(response.body)
-      parsed_response["data"].size.should eq(3)
-      parsed_response["data"][0]["origin"].should eq(links[0])
-      parsed_response["data"][1]["origin"].should eq(links[1])
-      parsed_response["data"][2]["origin"].should eq(links[2])
+      parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body)
+      parsed_response["data"].as(Array).size.should eq(3)
+
+      origins = parsed_response["data"].as(Array).map { |link| link["origin"] }
+      links[0..2].each do |link|
+        origins.should contain(link)
+      end
+      origins.should_not contain(links[3])
     end
 
     it "should return 401 - missing api key" do
@@ -169,16 +254,20 @@ describe "App::Controllers::Link" do
   end
 
   describe "Get" do
-    it "should return the specified link with click details" do
+    it "should return the specified link with limited click details" do
       link = "https://bing.com"
       test_user = create_test_user()
       test_link = create_test_link(test_user, link)
 
+      110.times do
+        create_test_click(test_link)
+      end
+
       get("/api/links/#{test_link.id}", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
 
       parsed_response = Hash(String, Hash(String, String | Int64 | Array(Hash(String, String)))).from_json(response.body)
       parsed_response["data"]["origin"].should eq(link)
-      parsed_response["data"]["clicks"].should be_a(Array(Hash(String, String)))
+      parsed_response["data"]["clicks"].as(Array).size.should eq(100)
     end
 
     it "should return 404 - link does not exist" do
@@ -200,6 +289,85 @@ describe "App::Controllers::Link" do
     end
   end
 
+  describe "Clicks" do
+    it "should return paginated clicks for a link" do
+      link = "https://example.com"
+      test_user = create_test_user()
+      test_link = create_test_link(test_user, link)
+
+      5.times do
+        create_test_click(test_link)
+      end
+
+      get("/api/links/#{test_link.id}/clicks", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
+
+      parsed_response = Hash(String, Array(Hash(String, String)) | Hash(String, Bool | String?)).from_json(response.body)
+      parsed_response["data"].as(Array).size.should eq(5)
+      parsed_response["pagination"].as(Hash)["has_more"].should be_false
+    end
+
+    it "should respect limit parameter" do
+      link = "https://example.com"
+      test_user = create_test_user()
+      test_link = create_test_link(test_user, link)
+
+      10.times do
+        create_test_click(test_link)
+      end
+
+      get("/api/links/#{test_link.id}/clicks?limit=3", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
+
+      parsed_response = Hash(String, Array(Hash(String, String)) | Hash(String, Bool | String?)).from_json(response.body)
+      parsed_response["data"].as(Array).size.should eq(3)
+      parsed_response["pagination"].as(Hash)["has_more"].should be_true
+      parsed_response["pagination"].as(Hash)["next"].should_not be_nil
+    end
+
+    it "should support cursor-based pagination" do
+      link = "https://example.com"
+      test_user = create_test_user()
+      test_link = create_test_link(test_user, link)
+
+      10.times do
+        create_test_click(test_link)
+      end
+
+      # Get first page
+      get("/api/links/#{test_link.id}/clicks?limit=3", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
+      first_page = Hash(String, Array(Hash(String, String)) | Hash(String, Bool | String?)).from_json(response.body)
+      cursor = first_page["pagination"].as(Hash)["next"]
+
+      # Get second page using cursor
+      get("/api/links/#{test_link.id}/clicks?limit=3&cursor=#{cursor}", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
+      second_page = Hash(String, Array(Hash(String, String)) | Hash(String, Bool | String?)).from_json(response.body)
+
+      # Ensure different clicks are returned
+      first_page_ids = first_page["data"].as(Array).map { |click| click["id"] }
+      second_page_ids = second_page["data"].as(Array).map { |click| click["id"] }
+
+      # Check that no IDs from first page appear in second page
+      (first_page_ids & second_page_ids).empty?.should be_true
+    end
+
+    it "should return 404 - link does not exist" do
+      test_user = create_test_user()
+
+      get("/api/links/nonexistent_id/clicks", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
+
+      expected = {"error" => "Resource not found"}.to_json
+      response.status_code.should eq(404)
+      response.body.should eq(expected)
+    end
+
+    it "should return 401 - missing api key" do
+      get("/api/links/1/clicks")
+
+      expected = {"error" => "Unauthorized access"}.to_json
+      response.status_code.should eq(401)
+      response.body.should eq(expected)
+    end
+  end
+
   describe "Update" do
     it "should update link url" do
       link = "https://github.com"

spec/integration/ping_spec.cr

@@ -4,7 +4,7 @@ describe "App::Controllers::Ping" do
   it "should return pong" do
     get "/api/ping"
 
-    expected = {"pong" => "ok"}.to_json
+    expected = {"data" => "pong"}.to_json
     response.body.should eq(expected)
   end
 end

spec/spec_helper.cr

@@ -1,4 +1,3 @@
-require "uuid"
 require "file_utils"
 
 require "spec-kemal"
@@ -24,7 +23,6 @@ end
 
 def create_test_user
   user = App::Models::User.new
-  user.id = UUID.v4.to_s
   user.name = "Tester"
   user.api_key = Random::Secure.urlsafe_base64()
 
@@ -39,8 +37,7 @@ end
 
 def create_test_link(user, url)
   link = App::Models::Link.new
-  link.id = UUID.v4.to_s
-  link.slug = App::Services::SlugService.shorten_url(url, user.id.to_s)
+  link.slug = App::Services::SlugService.shorten_url(url, user.id)
   link.url = url
   link.user = user
 
@@ -55,8 +52,27 @@ def create_test_link(user, url)
   link
 end
 
-def get_test_link(link_id)
-  query = App::Lib::Database::Query.where(id: link_id.as(String)).limit(1)
+def create_test_click(link)
+  click = App::Models::Click.new
+  click.user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0"
+  click.browser = "Firefox"
+  click.os = "Mac OS X"
+  click.referer = "example.com"
+  click.country = "US"
+  click.created_at = Time.utc
+  click.link = link
+  click.link_id = link.id
+
+  changeset = App::Lib::Database.insert(click)
+  unless changeset.valid?
+    error_messages = changeset.errors.map { |error| "#{error}" }.join(", ")
+    raise "Test click creation failed: #{error_messages}"
+  end
+  click
+end
+
+def get_test_link(link_id: Int64)
+  query = App::Lib::Database::Query.where(id: link_id).limit(1)
   link = App::Lib::Database.all(App::Models::Link, query, preload: [:clicks]).first?
 
   raise "Link not found" if link.nil?
@@ -64,6 +80,6 @@ def get_test_link(link_id)
   link
 end
 
-def delete_test_link(link_id)
+def delete_test_link(link_id: Int64)
   App::Lib::Database.raw_exec("DELETE FROM links WHERE id = (?)", link_id) # tempfix: Database.delete does not work
 end