refactor: link controller buffered channel to hold click data
This commit is contained in:
sjdonado
@@ -2,9 +2,11 @@
|
||||
[](https://hub.docker.com/r/sjdonado/bit)
|
||||
[](https://hub.docker.com/r/sjdonado/bit)
|
||||
|
||||
Lightweight URL shortener (API-only) with minimal resource requirements. Avg memory consumption under pressure is **60MiB**, single CPU core consumption around 30% (1K reqs/sec, latency 30ms, [benchmark](docs/SETUP.md#benchmark)).
|
||||
Lightweight URL shortener (API-only) with minimal resource requirements. Avg memory consumption under pressure is **70MiB**, single CPU core consumption below 60%.
|
||||
|
||||
Self-hosted with [Dokku](docs/SETUP.md#dokku) and [Docker Compose](docs/SETUP.md#docker-compose).
|
||||
Highly performant: 7K reqs/sec, latency 20ms (100000 request(s) using 125 connection(s), [benchmark](docs/SETUP.md#benchmark)).
|
||||
|
||||
Self-hosted: [Dokku](docs/SETUP.md#dokku), [Docker Compose](docs/SETUP.md#docker-compose).
|
||||
|
||||
Images available on [Docker Hub](https://hub.docker.com/r/sjdonado/bit/tags).
|
||||
|
||||
@@ -17,7 +19,7 @@ It is feature-complete by design: simple and reliable without unnecessary bloat.
|
||||
|
||||
## Minimum Requirements
|
||||
- 100MB disk space
|
||||
- 50MiB RAM
|
||||
- 80MiB RAM
|
||||
- x86_64 or ARM64
|
||||
|
||||
## Documentation
|
||||
|
||||
+87
-20
@@ -4,6 +4,82 @@ module App::Controllers
|
||||
include App::Lib
|
||||
include App::Services
|
||||
|
||||
# Buffered channel to hold click data
|
||||
@@click_channel = Channel(NamedTuple(
|
||||
link_id: Int64,
|
||||
remote_address: String,
|
||||
user_agent: String?,
|
||||
referer: String
|
||||
)).new(10000) # Buffer size
|
||||
|
||||
@@processor_started = begin
|
||||
spawn do
|
||||
batch_size = 125
|
||||
batch = [] of NamedTuple(
|
||||
link_id: Int64,
|
||||
remote_address: String,
|
||||
user_agent: String?,
|
||||
referer: String
|
||||
)
|
||||
|
||||
loop do
|
||||
select
|
||||
when click_data = @@click_channel.receive
|
||||
batch << click_data
|
||||
|
||||
# Collect clicks until we have a batch or a timeout
|
||||
if batch.size >= batch_size
|
||||
process_click_batch(batch)
|
||||
batch.clear
|
||||
end
|
||||
when timeout(0.5.seconds)
|
||||
# Process whatever we have after timeout
|
||||
unless batch.empty?
|
||||
process_click_batch(batch)
|
||||
batch.clear
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
true
|
||||
end
|
||||
|
||||
private def self.process_click_batch(batch)
|
||||
clicks = [] of App::Models::Click
|
||||
|
||||
batch.each do |click_data|
|
||||
begin
|
||||
client_ip = IpLookup.ip_from_address(click_data[:remote_address])
|
||||
family, _, _, os = UserAgent.parse(click_data[:user_agent] || "")
|
||||
|
||||
click = App::Models::Click.new
|
||||
click.link_id = click_data[:link_id]
|
||||
click.country = client_ip ? IpLookup.country(client_ip) : nil
|
||||
click.user_agent = click_data[:user_agent]
|
||||
click.browser = family
|
||||
click.os = os.try &.[0] # OS family
|
||||
click.referer = click_data[:referer]
|
||||
|
||||
clicks << click
|
||||
rescue ex
|
||||
Log.error { "Click data processing error: #{ex.message}" }
|
||||
end
|
||||
end
|
||||
|
||||
# Batch insert clicks if any were successfully processed
|
||||
unless clicks.empty?
|
||||
begin
|
||||
multi = Crecto::Multi.new
|
||||
clicks.each do |click|
|
||||
multi.insert(click)
|
||||
end
|
||||
Database.transaction(multi)
|
||||
rescue ex
|
||||
Log.error { "Batch click insertion error: #{ex.message}" }
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def self.redirect_handler
|
||||
->(env : HTTP::Server::Context) {
|
||||
link_id, url = Database.raw_query("SELECT id, url FROM links WHERE slug = (?) LIMIT 1", env.params.url["slug"]) do |result|
|
||||
@@ -16,26 +92,17 @@ module App::Controllers
|
||||
env.response.headers.add("Location", url)
|
||||
env.response.headers.add("X-Forwarded-For", remote_address)
|
||||
|
||||
spawn do
|
||||
begin
|
||||
client_ip = IpLookup.ip_from_address(remote_address)
|
||||
user_agent_str = env.request.headers["User-Agent"]?
|
||||
referer = env.request.headers["Referer"]?.try { |r| URI.parse(r).host rescue r } || env.params.query["utm_source"]? || "Direct"
|
||||
|
||||
family, _, _, os = user_agent_str ? UserAgent.parse(user_agent_str) : {nil, nil, nil, nil}
|
||||
|
||||
click = App::Models::Click.new
|
||||
click.link_id = link_id
|
||||
click.country = client_ip ? IpLookup.country(client_ip) : nil
|
||||
click.user_agent = user_agent_str
|
||||
click.browser = family
|
||||
click.os = os.try &.[0] # Access the first element of the os tuple (the family)
|
||||
click.referer = referer
|
||||
|
||||
Database.insert(click)
|
||||
rescue ex
|
||||
Log.error { "Click tracking error: #{ex.message}" }
|
||||
end
|
||||
begin
|
||||
@@click_channel.send({
|
||||
link_id: link_id,
|
||||
remote_address: remote_address,
|
||||
user_agent: env.request.headers["User-Agent"]?,
|
||||
referer: env.request.headers["Referer"]?.try { |r| URI.parse(r).host rescue r } || env.params.query["utm_source"]? || "Direct"
|
||||
})
|
||||
rescue Channel::ClosedError
|
||||
Log.error { "Click channel closed" }
|
||||
rescue ex
|
||||
Log.error { "Error queuing click: #{ex.message}" }
|
||||
end
|
||||
}
|
||||
end
|
||||
|
||||
+6
-7
@@ -8,7 +8,7 @@ require "file_utils"
|
||||
SERVER_URL = "http://localhost:4000"
|
||||
API_URL = "#{SERVER_URL}/api/links"
|
||||
API_KEY = "secure_api_key_1"
|
||||
NUMBER_OF_REQUESTS = 1000
|
||||
NUMBER_OF_REQUESTS = 100000
|
||||
|
||||
CONTAINER_NAME = "bit"
|
||||
STATS_FILE = "resource_usage.txt"
|
||||
@@ -128,7 +128,7 @@ def setup_containers
|
||||
rescue
|
||||
false
|
||||
end
|
||||
sleep 1.seconds
|
||||
sleep 2.seconds
|
||||
end
|
||||
end
|
||||
|
||||
@@ -157,7 +157,7 @@ def run_benchmark
|
||||
sleep 2.seconds
|
||||
process = Process.new(
|
||||
"bombardier",
|
||||
["-n", NUMBER_OF_REQUESTS.to_s, "-c", "30", "-l", "--disableKeepAlives", random_link],
|
||||
["-n", NUMBER_OF_REQUESTS.to_s, "-l", "--disableKeepAlives", random_link],
|
||||
output: Process::Redirect::Inherit,
|
||||
error: Process::Redirect::Inherit
|
||||
)
|
||||
@@ -175,6 +175,7 @@ end
|
||||
def analyze_resource_usage
|
||||
puts "Analyzing resource usage..."
|
||||
|
||||
sleep 2.seconds
|
||||
# Read stats directly from file for more accurate results
|
||||
if File.exists?(STATS_FILE)
|
||||
lines = File.read_lines(STATS_FILE)
|
||||
@@ -209,7 +210,6 @@ def analyze_resource_usage
|
||||
avg_cpu = total_cpu / lines.size
|
||||
avg_memory = total_memory / lines.size
|
||||
|
||||
# Format statistics summary
|
||||
stats_summary = <<-STATS
|
||||
**** Resource Usage Statistics ****
|
||||
Measurements: #{lines.size}
|
||||
@@ -220,12 +220,11 @@ def analyze_resource_usage
|
||||
|
||||
STATS
|
||||
|
||||
puts stats_summary
|
||||
|
||||
# Append summary to stats file
|
||||
File.open(STATS_FILE, "a") do |file|
|
||||
file.puts "\n" + stats_summary
|
||||
end
|
||||
|
||||
puts File.read(STATS_FILE)
|
||||
else
|
||||
puts "No resource usage data collected."
|
||||
end
|
||||
|
||||
+30
-51
@@ -126,64 +126,43 @@ Waiting for the application to be ready...
|
||||
Seeding the database...
|
||||
Checking seed results...
|
||||
Fetching all created links from /api/links...
|
||||
Selected link for benchmarking: http://localhost:4000/slug6268
|
||||
Selected link for benchmarking: http://localhost:4000/slug8558
|
||||
Starting benchmark with Bombardier...
|
||||
Bombarding http://localhost:4000/slug6268 for 59s using 30 connection(s)
|
||||
[==============================================================================================] 59s
|
||||
Bombarding http://localhost:4000/slug8558 with 100000 request(s) using 125 connection(s)
|
||||
100000 / 100000 [==============================================================] 100.00% 7126/s 14s
|
||||
Done!
|
||||
Statistics Avg Stdev Max
|
||||
Reqs/sec 1113.53 384.47 1642.40
|
||||
Latency 27.13ms 6.87ms 246.27ms
|
||||
Reqs/sec 7233.84 1873.69 13200.45
|
||||
Latency 17.31ms 1.44ms 44.42ms
|
||||
Latency Distribution
|
||||
50% 25.13ms
|
||||
75% 27.60ms
|
||||
90% 32.34ms
|
||||
95% 36.06ms
|
||||
99% 50.99ms
|
||||
50% 17.11ms
|
||||
75% 18.36ms
|
||||
90% 19.75ms
|
||||
95% 20.64ms
|
||||
99% 22.72ms
|
||||
HTTP codes:
|
||||
1xx - 0, 2xx - 0, 3xx - 65268, 4xx - 0, 5xx - 0
|
||||
1xx - 0, 2xx - 0, 3xx - 100000, 4xx - 0, 5xx - 0
|
||||
others - 0
|
||||
Throughput: 308.80KB/s
|
||||
Throughput: 1.97MB/s
|
||||
Benchmark completed successfully.
|
||||
Analyzing resource usage...
|
||||
**** Resource Usage Statistics ****
|
||||
Measurements: 31
|
||||
Average CPU Usage: 30.49%
|
||||
Average Memory Usage: 64.82 MiB
|
||||
Peak CPU Usage: 34.12%
|
||||
Peak Memory Usage: 65.45 MiB
|
||||
Cleanup completed. Resource usage data saved in resource_usage.txt
|
||||
~/p/bit> cat resource_usage.txt
|
||||
Timestamp CPU(%) Memory(MiB)
|
||||
1742499555 0.0 61.76
|
||||
1742499557 0.01 61.76
|
||||
1742499559 26.73 65.13
|
||||
1742499561 32.59 65.16
|
||||
1742499563 32.66 65.42
|
||||
1742499565 33.32 65.45
|
||||
1742499567 31.84 65.2
|
||||
1742499569 33.01 65.4
|
||||
1742499571 32.56 65.23
|
||||
1742499573 32.86 65.23
|
||||
1742499575 33.31 65.24
|
||||
1742499577 33.0 65.06
|
||||
1742499579 32.98 65.07
|
||||
1742499581 33.42 64.93
|
||||
1742499583 32.98 64.91
|
||||
1742499585 32.85 64.93
|
||||
1742499587 33.39 64.94
|
||||
1742499589 32.88 64.95
|
||||
1742499591 31.9 64.95
|
||||
1742499593 34.12 65.21
|
||||
1742499595 32.85 64.94
|
||||
1742499597 32.95 64.89
|
||||
1742499599 33.88 64.9
|
||||
1742499601 31.93 64.89
|
||||
1742499603 33.67 64.89
|
||||
1742499605 32.62 64.89
|
||||
1742499607 31.12 65.01
|
||||
1742499609 31.04 64.77
|
||||
1742499611 33.95 64.77
|
||||
1742499613 32.3 64.68
|
||||
1742499615 32.52 64.94
|
||||
1742732558 0.04 50.07
|
||||
1742732560 0.03 50.07
|
||||
1742732562 88.02 79.78
|
||||
1742732564 86.57 79.02
|
||||
1742732566 89.27 79.3
|
||||
1742732568 87.5 79.09
|
||||
1742732570 88.88 79.12
|
||||
1742732572 88.35 79.41
|
||||
1742732574 88.88 79.44
|
||||
1742732576 0.02 78.53
|
||||
|
||||
**** Resource Usage Statistics ****
|
||||
Measurements: 10
|
||||
Average CPU Usage: 61.76%
|
||||
Average Memory Usage: 73.38 MiB
|
||||
Peak CPU Usage: 89.27%
|
||||
Peak Memory Usage: 79.78 MiB
|
||||
Cleanup completed. Resource usage data saved in resource_usage.txt
|
||||
```
|
||||
|
||||
Reference in New Issue
Block a user