diff --git a/README.md b/README.md index 7cd2a8a..bf27256 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,11 @@ [![Docker Stars](https://img.shields.io/docker/stars/sjdonado/bit.svg)](https://hub.docker.com/r/sjdonado/bit) [![Docker Image Size](https://img.shields.io/docker/image-size/sjdonado/bit/latest)](https://hub.docker.com/r/sjdonado/bit) -Lightweight URL shortener (API-only) with minimal resource requirements. Avg memory consumption under pressure is **60MiB**, single CPU core consumption around 30% (1K reqs/sec, latency 30ms, [benchmark](docs/SETUP.md#benchmark)). +Lightweight URL shortener (API-only) with minimal resource requirements. Avg memory consumption under pressure is **70MiB**, single CPU core consumption below 60%. -Self-hosted with [Dokku](docs/SETUP.md#dokku) and [Docker Compose](docs/SETUP.md#docker-compose). +Highly performant: 7K reqs/sec, latency 20ms (100000 request(s) using 125 connection(s), [benchmark](docs/SETUP.md#benchmark)). + +Self-hosted: [Dokku](docs/SETUP.md#dokku), [Docker Compose](docs/SETUP.md#docker-compose). Images available on [Docker Hub](https://hub.docker.com/r/sjdonado/bit/tags). @@ -17,7 +19,7 @@ It is feature-complete by design: simple and reliable without unnecessary bloat. ## Minimum Requirements - 100MB disk space -- 50MiB RAM +- 80MiB RAM - x86_64 or ARM64 ## Documentation diff --git a/app/controllers/click.cr b/app/controllers/click.cr index db9e3e9..3f2da18 100644 --- a/app/controllers/click.cr +++ b/app/controllers/click.cr @@ -4,6 +4,82 @@ module App::Controllers include App::Lib include App::Services + # Buffered channel to hold click data + @@click_channel = Channel(NamedTuple( + link_id: Int64, + remote_address: String, + user_agent: String?, + referer: String + )).new(10000) # Buffer size + + @@processor_started = begin + spawn do + batch_size = 125 + batch = [] of NamedTuple( + link_id: Int64, + remote_address: String, + user_agent: String?, + referer: String + ) + + loop do + select + when click_data = @@click_channel.receive + batch << click_data + + # Collect clicks until we have a batch or a timeout + if batch.size >= batch_size + process_click_batch(batch) + batch.clear + end + when timeout(0.5.seconds) + # Process whatever we have after timeout + unless batch.empty? + process_click_batch(batch) + batch.clear + end + end + end + end + true + end + + private def self.process_click_batch(batch) + clicks = [] of App::Models::Click + + batch.each do |click_data| + begin + client_ip = IpLookup.ip_from_address(click_data[:remote_address]) + family, _, _, os = UserAgent.parse(click_data[:user_agent] || "") + + click = App::Models::Click.new + click.link_id = click_data[:link_id] + click.country = client_ip ? IpLookup.country(client_ip) : nil + click.user_agent = click_data[:user_agent] + click.browser = family + click.os = os.try &.[0] # OS family + click.referer = click_data[:referer] + + clicks << click + rescue ex + Log.error { "Click data processing error: #{ex.message}" } + end + end + + # Batch insert clicks if any were successfully processed + unless clicks.empty? + begin + multi = Crecto::Multi.new + clicks.each do |click| + multi.insert(click) + end + Database.transaction(multi) + rescue ex + Log.error { "Batch click insertion error: #{ex.message}" } + end + end + end + def self.redirect_handler ->(env : HTTP::Server::Context) { link_id, url = Database.raw_query("SELECT id, url FROM links WHERE slug = (?) LIMIT 1", env.params.url["slug"]) do |result| @@ -16,26 +92,17 @@ module App::Controllers env.response.headers.add("Location", url) env.response.headers.add("X-Forwarded-For", remote_address) - spawn do - begin - client_ip = IpLookup.ip_from_address(remote_address) - user_agent_str = env.request.headers["User-Agent"]? - referer = env.request.headers["Referer"]?.try { |r| URI.parse(r).host rescue r } || env.params.query["utm_source"]? || "Direct" - - family, _, _, os = user_agent_str ? UserAgent.parse(user_agent_str) : {nil, nil, nil, nil} - - click = App::Models::Click.new - click.link_id = link_id - click.country = client_ip ? IpLookup.country(client_ip) : nil - click.user_agent = user_agent_str - click.browser = family - click.os = os.try &.[0] # Access the first element of the os tuple (the family) - click.referer = referer - - Database.insert(click) - rescue ex - Log.error { "Click tracking error: #{ex.message}" } - end + begin + @@click_channel.send({ + link_id: link_id, + remote_address: remote_address, + user_agent: env.request.headers["User-Agent"]?, + referer: env.request.headers["Referer"]?.try { |r| URI.parse(r).host rescue r } || env.params.query["utm_source"]? || "Direct" + }) + rescue Channel::ClosedError + Log.error { "Click channel closed" } + rescue ex + Log.error { "Error queuing click: #{ex.message}" } end } end diff --git a/benchmark.cr b/benchmark.cr index 75672f4..18ac1b1 100755 --- a/benchmark.cr +++ b/benchmark.cr @@ -8,7 +8,7 @@ require "file_utils" SERVER_URL = "http://localhost:4000" API_URL = "#{SERVER_URL}/api/links" API_KEY = "secure_api_key_1" -NUMBER_OF_REQUESTS = 1000 +NUMBER_OF_REQUESTS = 100000 CONTAINER_NAME = "bit" STATS_FILE = "resource_usage.txt" @@ -128,7 +128,7 @@ def setup_containers rescue false end - sleep 1.seconds + sleep 2.seconds end end @@ -157,7 +157,7 @@ def run_benchmark sleep 2.seconds process = Process.new( "bombardier", - ["-n", NUMBER_OF_REQUESTS.to_s, "-c", "30", "-l", "--disableKeepAlives", random_link], + ["-n", NUMBER_OF_REQUESTS.to_s, "-l", "--disableKeepAlives", random_link], output: Process::Redirect::Inherit, error: Process::Redirect::Inherit ) @@ -175,6 +175,7 @@ end def analyze_resource_usage puts "Analyzing resource usage..." + sleep 2.seconds # Read stats directly from file for more accurate results if File.exists?(STATS_FILE) lines = File.read_lines(STATS_FILE) @@ -209,7 +210,6 @@ def analyze_resource_usage avg_cpu = total_cpu / lines.size avg_memory = total_memory / lines.size - # Format statistics summary stats_summary = <<-STATS **** Resource Usage Statistics **** Measurements: #{lines.size} @@ -220,12 +220,11 @@ def analyze_resource_usage STATS - puts stats_summary - - # Append summary to stats file File.open(STATS_FILE, "a") do |file| file.puts "\n" + stats_summary end + + puts File.read(STATS_FILE) else puts "No resource usage data collected." end diff --git a/docs/SETUP.md b/docs/SETUP.md index 954e80f..3fbbce2 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -126,64 +126,43 @@ Waiting for the application to be ready... Seeding the database... Checking seed results... Fetching all created links from /api/links... -Selected link for benchmarking: http://localhost:4000/slug6268 +Selected link for benchmarking: http://localhost:4000/slug8558 Starting benchmark with Bombardier... -Bombarding http://localhost:4000/slug6268 for 59s using 30 connection(s) -[==============================================================================================] 59s +Bombarding http://localhost:4000/slug8558 with 100000 request(s) using 125 connection(s) + 100000 / 100000 [==============================================================] 100.00% 7126/s 14s Done! Statistics Avg Stdev Max - Reqs/sec 1113.53 384.47 1642.40 - Latency 27.13ms 6.87ms 246.27ms + Reqs/sec 7233.84 1873.69 13200.45 + Latency 17.31ms 1.44ms 44.42ms Latency Distribution - 50% 25.13ms - 75% 27.60ms - 90% 32.34ms - 95% 36.06ms - 99% 50.99ms + 50% 17.11ms + 75% 18.36ms + 90% 19.75ms + 95% 20.64ms + 99% 22.72ms HTTP codes: - 1xx - 0, 2xx - 0, 3xx - 65268, 4xx - 0, 5xx - 0 + 1xx - 0, 2xx - 0, 3xx - 100000, 4xx - 0, 5xx - 0 others - 0 - Throughput: 308.80KB/s + Throughput: 1.97MB/s Benchmark completed successfully. Analyzing resource usage... -**** Resource Usage Statistics **** - Measurements: 31 - Average CPU Usage: 30.49% - Average Memory Usage: 64.82 MiB - Peak CPU Usage: 34.12% - Peak Memory Usage: 65.45 MiB -Cleanup completed. Resource usage data saved in resource_usage.txt -~/p/bit> cat resource_usage.txt Timestamp CPU(%) Memory(MiB) -1742499555 0.0 61.76 -1742499557 0.01 61.76 -1742499559 26.73 65.13 -1742499561 32.59 65.16 -1742499563 32.66 65.42 -1742499565 33.32 65.45 -1742499567 31.84 65.2 -1742499569 33.01 65.4 -1742499571 32.56 65.23 -1742499573 32.86 65.23 -1742499575 33.31 65.24 -1742499577 33.0 65.06 -1742499579 32.98 65.07 -1742499581 33.42 64.93 -1742499583 32.98 64.91 -1742499585 32.85 64.93 -1742499587 33.39 64.94 -1742499589 32.88 64.95 -1742499591 31.9 64.95 -1742499593 34.12 65.21 -1742499595 32.85 64.94 -1742499597 32.95 64.89 -1742499599 33.88 64.9 -1742499601 31.93 64.89 -1742499603 33.67 64.89 -1742499605 32.62 64.89 -1742499607 31.12 65.01 -1742499609 31.04 64.77 -1742499611 33.95 64.77 -1742499613 32.3 64.68 -1742499615 32.52 64.94 +1742732558 0.04 50.07 +1742732560 0.03 50.07 +1742732562 88.02 79.78 +1742732564 86.57 79.02 +1742732566 89.27 79.3 +1742732568 87.5 79.09 +1742732570 88.88 79.12 +1742732572 88.35 79.41 +1742732574 88.88 79.44 +1742732576 0.02 78.53 + +**** Resource Usage Statistics **** + Measurements: 10 + Average CPU Usage: 61.76% + Average Memory Usage: 73.38 MiB + Peak CPU Usage: 89.27% + Peak Memory Usage: 79.78 MiB +Cleanup completed. Resource usage data saved in resource_usage.txt ```