refactor: link controller buffered channel to hold click data

README.md

@@ -2,9 +2,11 @@
 [![Docker Stars](https://img.shields.io/docker/stars/sjdonado/bit.svg)](https://hub.docker.com/r/sjdonado/bit)
 [![Docker Image Size](https://img.shields.io/docker/image-size/sjdonado/bit/latest)](https://hub.docker.com/r/sjdonado/bit)
 
-Lightweight URL shortener (API-only) with minimal resource requirements. Avg memory consumption under pressure is **60MiB**, single CPU core consumption around 30% (1K reqs/sec, latency 30ms, [benchmark](docs/SETUP.md#benchmark)).
+Lightweight URL shortener (API-only) with minimal resource requirements. Avg memory consumption under pressure is **70MiB**, single CPU core consumption below 60%.
 
-Self-hosted with [Dokku](docs/SETUP.md#dokku) and [Docker Compose](docs/SETUP.md#docker-compose).
+Highly performant: 7K reqs/sec, latency 20ms (100000 request(s) using 125 connection(s), [benchmark](docs/SETUP.md#benchmark)).
+
+Self-hosted: [Dokku](docs/SETUP.md#dokku), [Docker Compose](docs/SETUP.md#docker-compose).
 
 Images available on [Docker Hub](https://hub.docker.com/r/sjdonado/bit/tags).
 
@@ -17,7 +19,7 @@ It is feature-complete by design: simple and reliable without unnecessary bloat.
 
 ## Minimum Requirements
 - 100MB disk space
-- 50MiB RAM
+- 80MiB RAM
 - x86_64 or ARM64
 
 ## Documentation

app/controllers/click.cr

@@ -4,6 +4,82 @@ module App::Controllers
     include App::Lib
     include App::Services
 
+    # Buffered channel to hold click data
+    @@click_channel = Channel(NamedTuple(
+      link_id: Int64,
+      remote_address: String,
+      user_agent: String?,
+      referer: String
+    )).new(10000) # Buffer size
+
+    @@processor_started = begin
+      spawn do
+        batch_size = 125
+        batch = [] of NamedTuple(
+          link_id: Int64,
+          remote_address: String,
+          user_agent: String?,
+          referer: String
+        )
+
+        loop do
+          select
+          when click_data = @@click_channel.receive
+            batch << click_data
+
+            # Collect clicks until we have a batch or a timeout
+            if batch.size >= batch_size
+              process_click_batch(batch)
+              batch.clear
+            end
+          when timeout(0.5.seconds)
+            # Process whatever we have after timeout
+            unless batch.empty?
+              process_click_batch(batch)
+              batch.clear
+            end
+          end
+        end
+      end
+      true
+    end
+
+    private def self.process_click_batch(batch)
+      clicks = [] of App::Models::Click
+
+      batch.each do |click_data|
+        begin
+          client_ip = IpLookup.ip_from_address(click_data[:remote_address])
+          family, _, _, os = UserAgent.parse(click_data[:user_agent] || "")
+
+          click = App::Models::Click.new
+          click.link_id = click_data[:link_id]
+          click.country = client_ip ? IpLookup.country(client_ip) : nil
+          click.user_agent = click_data[:user_agent]
+          click.browser = family
+          click.os = os.try &.[0]  # OS family
+          click.referer = click_data[:referer]
+
+          clicks << click
+        rescue ex
+          Log.error { "Click data processing error: #{ex.message}" }
+        end
+      end
+
+      # Batch insert clicks if any were successfully processed
+       unless clicks.empty?
+        begin
+          multi = Crecto::Multi.new
+          clicks.each do |click|
+            multi.insert(click)
+          end
+          Database.transaction(multi)
+        rescue ex
+          Log.error { "Batch click insertion error: #{ex.message}" }
+        end
+      end
+    end
+
     def self.redirect_handler
       ->(env : HTTP::Server::Context) {
         link_id, url = Database.raw_query("SELECT id, url FROM links WHERE slug = (?) LIMIT 1", env.params.url["slug"]) do |result|
@@ -16,26 +92,17 @@ module App::Controllers
         env.response.headers.add("Location", url)
         env.response.headers.add("X-Forwarded-For", remote_address)
 
-        spawn do
+        begin
-          begin
+          @@click_channel.send({
-            client_ip = IpLookup.ip_from_address(remote_address)
+            link_id: link_id,
-            user_agent_str = env.request.headers["User-Agent"]?
+            remote_address: remote_address,
-            referer = env.request.headers["Referer"]?.try { |r| URI.parse(r).host rescue r } || env.params.query["utm_source"]? || "Direct"
+            user_agent: env.request.headers["User-Agent"]?,
-
+            referer: env.request.headers["Referer"]?.try { |r| URI.parse(r).host rescue r } || env.params.query["utm_source"]? || "Direct"
-            family, _, _, os = user_agent_str ? UserAgent.parse(user_agent_str) : {nil, nil, nil, nil}
+          })
-
+        rescue Channel::ClosedError
-            click = App::Models::Click.new
+          Log.error { "Click channel closed" }
-            click.link_id = link_id
+        rescue ex
-            click.country = client_ip ? IpLookup.country(client_ip) : nil
+          Log.error { "Error queuing click: #{ex.message}" }
-            click.user_agent = user_agent_str
-            click.browser = family
-            click.os = os.try &.[0]  # Access the first element of the os tuple (the family)
-            click.referer = referer
-
-            Database.insert(click)
-          rescue ex
-            Log.error { "Click tracking error: #{ex.message}" }
-          end
         end
       }
     end

benchmark.cr

@@ -8,7 +8,7 @@ require "file_utils"
 SERVER_URL = "http://localhost:4000"
 API_URL = "#{SERVER_URL}/api/links"
 API_KEY = "secure_api_key_1"
-NUMBER_OF_REQUESTS = 1000
+NUMBER_OF_REQUESTS = 100000
 
 CONTAINER_NAME = "bit"
 STATS_FILE = "resource_usage.txt"
@@ -128,7 +128,7 @@ def setup_containers
         rescue
           false
         end
-    sleep 1.seconds
+    sleep 2.seconds
   end
 end
 
@@ -157,7 +157,7 @@ def run_benchmark
   sleep 2.seconds
   process = Process.new(
     "bombardier",
-    ["-n", NUMBER_OF_REQUESTS.to_s, "-c", "30", "-l", "--disableKeepAlives", random_link],
+    ["-n", NUMBER_OF_REQUESTS.to_s, "-l", "--disableKeepAlives", random_link],
     output: Process::Redirect::Inherit,
     error: Process::Redirect::Inherit
   )
@@ -175,6 +175,7 @@ end
 def analyze_resource_usage
   puts "Analyzing resource usage..."
 
+  sleep 2.seconds
   # Read stats directly from file for more accurate results
   if File.exists?(STATS_FILE)
     lines = File.read_lines(STATS_FILE)
@@ -209,7 +210,6 @@ def analyze_resource_usage
       avg_cpu = total_cpu / lines.size
       avg_memory = total_memory / lines.size
 
-      # Format statistics summary
       stats_summary = <<-STATS
       **** Resource Usage Statistics ****
         Measurements: #{lines.size}
@@ -220,12 +220,11 @@ def analyze_resource_usage
 
       STATS
 
-      puts stats_summary
-
-      # Append summary to stats file
       File.open(STATS_FILE, "a") do |file|
         file.puts "\n" + stats_summary
       end
+
+      puts File.read(STATS_FILE)
     else
       puts "No resource usage data collected."
     end

docs/SETUP.md

@@ -126,64 +126,43 @@ Waiting for the application to be ready...
 Seeding the database...
 Checking seed results...
 Fetching all created links from /api/links...
-Selected link for benchmarking: http://localhost:4000/slug6268
+Selected link for benchmarking: http://localhost:4000/slug8558
 Starting benchmark with Bombardier...
-Bombarding http://localhost:4000/slug6268 for 59s using 30 connection(s)
+Bombarding http://localhost:4000/slug8558 with 100000 request(s) using 125 connection(s)
-[==============================================================================================] 59s
+ 100000 / 100000 [==============================================================] 100.00% 7126/s 14s
 Done!
 Statistics        Avg      Stdev        Max
-  Reqs/sec      1113.53     384.47    1642.40
+  Reqs/sec      7233.84    1873.69   13200.45
-  Latency       27.13ms     6.87ms   246.27ms
+  Latency       17.31ms     1.44ms    44.42ms
   Latency Distribution
-     50%    25.13ms
+     50%    17.11ms
-     75%    27.60ms
+     75%    18.36ms
-     90%    32.34ms
+     90%    19.75ms
-     95%    36.06ms
+     95%    20.64ms
-     99%    50.99ms
+     99%    22.72ms
   HTTP codes:
-    1xx - 0, 2xx - 0, 3xx - 65268, 4xx - 0, 5xx - 0
+    1xx - 0, 2xx - 0, 3xx - 100000, 4xx - 0, 5xx - 0
     others - 0
-  Throughput:   308.80KB/s
+  Throughput:     1.97MB/s
 Benchmark completed successfully.
 Analyzing resource usage...
-**** Resource Usage Statistics ****
-  Measurements: 31
-  Average CPU Usage: 30.49%
-  Average Memory Usage: 64.82 MiB
-  Peak CPU Usage: 34.12%
-  Peak Memory Usage: 65.45 MiB
-Cleanup completed. Resource usage data saved in resource_usage.txt
-~/p/bit> cat resource_usage.txt
 Timestamp       CPU(%)  Memory(MiB)
-1742499555      0.0     61.76
+1742732558      0.04    50.07
-1742499557      0.01    61.76
+1742732560      0.03    50.07
-1742499559      26.73   65.13
+1742732562      88.02   79.78
-1742499561      32.59   65.16
+1742732564      86.57   79.02
-1742499563      32.66   65.42
+1742732566      89.27   79.3
-1742499565      33.32   65.45
+1742732568      87.5    79.09
-1742499567      31.84   65.2
+1742732570      88.88   79.12
-1742499569      33.01   65.4
+1742732572      88.35   79.41
-1742499571      32.56   65.23
+1742732574      88.88   79.44
-1742499573      32.86   65.23
+1742732576      0.02    78.53
-1742499575      33.31   65.24
+
-1742499577      33.0    65.06
+**** Resource Usage Statistics ****
-1742499579      32.98   65.07
+  Measurements: 10
-1742499581      33.42   64.93
+  Average CPU Usage: 61.76%
-1742499583      32.98   64.91
+  Average Memory Usage: 73.38 MiB
-1742499585      32.85   64.93
+  Peak CPU Usage: 89.27%
-1742499587      33.39   64.94
+  Peak Memory Usage: 79.78 MiB
-1742499589      32.88   64.95
+Cleanup completed. Resource usage data saved in resource_usage.txt
-1742499591      31.9    64.95
-1742499593      34.12   65.21
-1742499595      32.85   64.94
-1742499597      32.95   64.89
-1742499599      33.88   64.9
-1742499601      31.93   64.89
-1742499603      33.67   64.89
-1742499605      32.62   64.89
-1742499607      31.12   65.01
-1742499609      31.04   64.77
-1742499611      33.95   64.77
-1742499613      32.3    64.68
-1742499615      32.52   64.94
 ```