aheh/bit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #7 from sjdonado/refactor/benchmark-native-and-updates

Browse Source 
Refactor/benchmark native and updates
This commit is contained in:
Juan Rodriguez Donado
2025-11-02 10:59:57 +01:00
committed by GitHub
parent d0412d802b fef015ce53
commit 353cf68852
20 changed files with 1162 additions and 659 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/deploy-docs.yml
+33
View File
@@ -0,0 +1,33 @@
name: Deploy API Documentation
on:
push:
branches:
- master
paths:
- 'docs/openapi.yaml'
- '.github/workflows/deploy-docs.yml'
workflow_dispatch:
permissions:
contents: write
jobs:
deploy:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Generate Swagger UI
uses: Legion2/swagger-ui-action@v1
with:
output: swagger-ui
spec-file: docs/openapi.yaml
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Deploy to GitHub Pages
uses: peaceiris/actions-gh-pages@v3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: swagger-ui
.github/workflows/publish.yml
+9 -42
View File
@@ -8,12 +8,9 @@ on:
types: [published] types: [published]
jobs: jobs:
build-platforms: build-and-push:
name: Build Platforms name: Build and Push Multi-Platform
runs-on: ubuntu-latest runs-on: ubuntu-latest
strategy:
matrix:
platform: [linux/amd64, linux/arm64]
permissions: permissions:
contents: read contents: read
packages: write packages: write
@@ -22,8 +19,11 @@ jobs:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx - name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2 uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub - name: Login to Docker Hub
uses: docker/login-action@v3 uses: docker/login-action@v3
@@ -40,45 +40,12 @@ jobs:
echo "version=latest" >> $GITHUB_OUTPUT echo "version=latest" >> $GITHUB_OUTPUT
fi fi
- name: Build and push platform image - name: Build and push multi-platform image
uses: docker/build-push-action@v5 uses: docker/build-push-action@v5
env:
CRYSTAL_WORKERS: ${{ matrix.platform == 'linux/amd64' && 4 || 2 }}
with: with:
context: . context: .
platforms: ${{ matrix.platform }} platforms: linux/amd64,linux/arm64
push: true push: true
tags: | tags: sjdonado/bit:${{ steps.version.outputs.version }}
sjdonado/bit:${{ steps.version.outputs.version }}-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
build-args: |
TARGETARCH=${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
cache-from: type=gha cache-from: type=gha
cache-to: type=gha,mode=max cache-to: type=gha,mode=max
create-manifest:
name: Create Manifest
runs-on: ubuntu-latest
needs: build-platforms
permissions:
packages: write
steps:
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Determine version
id: version
run: |
if [ "${{ github.event_name }}" = "release" ]; then
echo "version=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT
else
echo "version=latest" >> $GITHUB_OUTPUT
fi
- name: Create manifest
run: |
docker buildx imagetools create \
-t sjdonado/bit:${{ steps.version.outputs.version }} \
sjdonado/bit:${{ steps.version.outputs.version }}-{amd64,arm64}
.github/workflows/update-parsers.yml
+63
View File
@@ -0,0 +1,63 @@
name: Update Parsers
on:
schedule:
# Run every two weeks on Sunday at 00:00 UTC (1st and 3rd Sunday of each month)
- cron: '0 0 1-7,15-21 * 0'
workflow_dispatch: # Allow manual trigger
jobs:
update-parsers:
runs-on: ubuntu-latest
permissions:
contents: write
pull-requests: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install Crystal
uses: crystal-lang/install-crystal@v1
- name: Install dependencies
run: shards install
- name: Build CLI
run: crystal build scripts/cli.cr -o cli
- name: Update parsers
run: ./cli --update-parsers
- name: Check for changes
id: changes
run: |
if [ -n "$(git status --porcelain)" ]; then
echo "has_changes=true" >> $GITHUB_OUTPUT
else
echo "has_changes=false" >> $GITHUB_OUTPUT
fi
- name: Create Pull Request
if: steps.changes.outputs.has_changes == 'true'
uses: peter-evans/create-pull-request@v6
with:
token: ${{ secrets.GITHUB_TOKEN }}
commit-message: 'chore: update parsers'
title: 'chore: Update UA regexes and GeoLite2 database'
body: |
## Automated Parser Update
This PR updates the following data files:
- User Agent parsing regexes
- GeoLite2 database
**Triggered by**: Scheduled workflow (runs every 2 weeks)
**Date**: ${{ github.event.repository.updated_at }}
Please review the changes and merge if everything looks good.
branch: automated/update-parsers
delete-branch: true
labels: |
dependencies
automated
.gitignore
+2 -1
View File
@@ -7,4 +7,5 @@
/sqlite/ /sqlite/
.env.production .env.production
resource_usage.* *.log
bit
Dockerfile
+18 -22
View File
@@ -1,41 +1,37 @@
FROM debian:bookworm-slim AS build FROM alpine:edge AS build
ARG TARGETARCH
ENV ENV=production ENV ENV=production
WORKDIR /usr/src/app WORKDIR /usr/src/app
RUN apt-get update && apt-get install -y \ RUN apk add --no-cache \
curl \
gnupg \
ca-certificates \
&& mkdir -p /etc/apt/keyrings \
&& curl -fsSL https://packagecloud.io/84codes/crystal/gpgkey | gpg --dearmor > /etc/apt/trusted.gpg.d/84codes_crystal.gpg \
&& echo "deb [signed-by=/etc/apt/trusted.gpg.d/84codes_crystal.gpg] https://packagecloud.io/84codes/crystal/debian/ bookworm main" > /etc/apt/sources.list.d/84codes_crystal.list
RUN apt-get update && apt-get install -y \
crystal \ crystal \
libssl-dev \ shards \
libyaml-dev \ openssl-dev \
libsqlite3-dev \ yaml-dev \
sqlite-dev \
libevent-dev \ libevent-dev \
&& rm -rf /var/lib/apt/lists/* tzdata
COPY . . COPY . .
RUN shards install --production RUN shards install --production
RUN shards build --release --no-debug --progress --stats RUN shards build --release --no-debug --progress --stats
FROM debian:bookworm-slim AS runtime FROM alpine:latest AS runtime
ENV ENV=production ENV ENV=production
WORKDIR /usr/src/app WORKDIR /usr/src/app
RUN apt-get update && apt-get install -y \ RUN apk add --no-cache \
libssl3 \ gc-dev \
libyaml-0-2 \ pcre2 \
sqlite3 \ libevent \
libevent-2.1-7 \ sqlite-libs \
&& rm -rf /var/lib/apt/lists/* openssl \
yaml \
gmp \
libgcc \
tzdata
RUN mkdir -p sqlite RUN mkdir -p sqlite
README.md
+3 -3
View File
@@ -2,9 +2,9 @@
[![Docker Stars](https://img.shields.io/docker/stars/sjdonado/bit.svg)](https://hub.docker.com/r/sjdonado/bit) [![Docker Stars](https://img.shields.io/docker/stars/sjdonado/bit.svg)](https://hub.docker.com/r/sjdonado/bit)
[![Docker Image Size](https://img.shields.io/docker/image-size/sjdonado/bit/latest)](https://hub.docker.com/r/sjdonado/bit) [![Docker Image Size](https://img.shields.io/docker/image-size/sjdonado/bit/latest)](https://hub.docker.com/r/sjdonado/bit)
Lightweight URL shortener (API-only) with minimal resource requirements. Avg memory consumption **30MiB**, avg CPU load 20%. Lightweight URL shortener (API-only) with minimal resource requirements.
Highly performant: **7.9k req/sec**, latency 15.8ms (100k requests using 125 connections, [benchmark](docs/SETUP.md#benchmark)). Highly performant: **11k req/sec**, latency 11ms, 40MiB avg memory usage (100k requests using 125 connections, [benchmark](docs/SETUP.md#benchmark)).
Self-hosted: [Dokku](docs/SETUP.md#dokku), [Docker Compose](docs/SETUP.md#docker-compose). Self-hosted: [Dokku](docs/SETUP.md#dokku), [Docker Compose](docs/SETUP.md#docker-compose).
@@ -20,7 +20,7 @@ It is feature-complete by design: simple and reliable without unnecessary bloat.
## Recommented requirements ## Recommented requirements
- 100MB disk space - 100MB disk space
- 70MiB RAM - 100MiB RAM
- x86_64 or ARM64 - x86_64 or ARM64
## Documentation ## Documentation
app/controllers/click.cr
+22 -95
View File
@@ -4,90 +4,6 @@ module App::Controllers
include App::Lib include App::Lib
include App::Services include App::Services
# Buffered channel to hold click data
@@click_channel = Channel(NamedTuple(
link_id: Int64,
remote_address: String,
user_agent: String?,
referer: String
)).new(1024)
@@processor_started = begin
spawn do
batch = [] of NamedTuple(
link_id: Int64,
remote_address: String,
user_agent: String?,
referer: String
)
batch_size = 32
min_batch = 32
max_batch = 512
scaling_factor = 1.5
loop do
select
when click_data = @@click_channel.receive
batch << click_data
if batch.size >= batch_size
process_click_batch(batch)
batch.clear
# Increase batch size when reaching capacity
batch_size = (batch_size * scaling_factor).to_i.clamp(min_batch, max_batch)
end
when timeout(1.seconds)
unless batch.empty?
process_click_batch(batch)
batch.clear
batch_size = (batch_size / scaling_factor).to_i.clamp(min_batch, max_batch)
else
# Reset to default if idle
batch_size = min_batch unless batch_size == min_batch
end
end
end
end
true
end
private def self.process_click_batch(batch)
clicks = [] of App::Models::Click
batch.each do |click_data|
begin
client_ip = IpLookup.ip_from_address(click_data[:remote_address])
family, _, _, os = UserAgent.parse(click_data[:user_agent] || "")
click = App::Models::Click.new
click.link_id = click_data[:link_id]
click.country = client_ip ? IpLookup.country(client_ip) : nil
click.user_agent = click_data[:user_agent]
click.browser = family
click.os = os.try &.[0] # OS family
click.referer = click_data[:referer]
clicks << click
rescue ex
Log.error { "Click data processing error: #{ex.message}" }
end
end
# Batch insert clicks if any were successfully processed
unless clicks.empty?
begin
multi = Crecto::Multi.new
clicks.each do |click|
multi.insert(click)
end
Database.transaction(multi)
rescue ex
Log.error { "Batch click insertion error: #{ex.message}" }
end
end
end
def self.redirect_handler def self.redirect_handler
->(env : HTTP::Server::Context) { ->(env : HTTP::Server::Context) {
link_id, url = Database.raw_query("SELECT id, url FROM links WHERE slug = (?) LIMIT 1", env.params.url["slug"]) do |result| link_id, url = Database.raw_query("SELECT id, url FROM links WHERE slug = (?) LIMIT 1", env.params.url["slug"]) do |result|
@@ -96,21 +12,32 @@ module App::Controllers
remote_address = env.request.headers["Cf-Connecting-Ip"]? || env.request.remote_address.to_s remote_address = env.request.headers["Cf-Connecting-Ip"]? || env.request.remote_address.to_s
# Send redirect immediately
env.response.status_code = 301 env.response.status_code = 301
env.response.headers.add("Location", url) env.response.headers.add("Location", url)
env.response.headers.add("X-Forwarded-For", remote_address) env.response.headers.add("X-Forwarded-For", remote_address)
if user_agent = env.request.headers["User-Agent"]?
env.response.headers.add("User-Agent", user_agent)
end
begin # non-blocking click proccessing
@@click_channel.send({ spawn do
link_id: link_id, begin
remote_address: remote_address, client_ip = IpLookup.ip_from_address(remote_address)
user_agent: env.request.headers["User-Agent"]?, family, _, _, os = UserAgent.parse(env.request.headers["User-Agent"]? || "")
referer: env.request.headers["Referer"]?.try { |r| URI.parse(r).host rescue r } || env.params.query["utm_source"]? || "Direct"
}) click = App::Models::Click.new
rescue Channel::ClosedError click.link_id = link_id
Log.error { "Click channel closed" } click.country = client_ip ? IpLookup.country(client_ip) : nil
rescue ex click.user_agent = env.request.headers["User-Agent"]?
Log.error { "Error queuing click: #{ex.message}" } click.browser = family
click.os = os.try &.[0]
click.referer = env.request.headers["Referer"]?.try { |r| URI.parse(r).host rescue r } || env.params.query["utm_source"]? || "Direct"
Database.insert(click)
rescue ex
Log.error { "Click tracking error: #{ex.message}" }
end
end end
} }
end end
app/controllers/link.cr
+1 -1
View File
@@ -145,7 +145,7 @@ module App::Controllers
"data" => items.map { |item| yield item }, "data" => items.map { |item| yield item },
"pagination" => { "pagination" => {
"has_more" => has_more, "has_more" => has_more,
"next_cursor" => next_cursor "next" => next_cursor
} }
}) })
end end
app/lib/controller.cr
+1 -1
View File
@@ -23,7 +23,7 @@ module App::Lib
missing_fields = required_fields.reject { |field| json_params.has_key?(field) } missing_fields = required_fields.reject { |field| json_params.has_key?(field) }
unless missing_fields.empty? unless missing_fields.empty?
error_message = missing_fields.join(", ") + " required" error_message = "#{missing_fields.first}: Required field"
raise App::BadRequestException.new(@env, error_message) raise App::BadRequestException.new(@env, error_message)
end end
app/services/cli.cr
+1 -1
View File
@@ -73,7 +73,7 @@ module App::Services::Cli
end end
end end
def self.download_geolite_db def self.update_geolite_db
puts "Downloading GeoLite2 Country database..." puts "Downloading GeoLite2 Country database..."
FileUtils.mkdir_p("data") FileUtils.mkdir_p("data")
benchmark.cr
-257
View File
@@ -1,257 +0,0 @@
#!/usr/bin/env crystal
require "http/client"
require "json"
require "file_utils"
# Configuration variables
SERVER_URL = "http://localhost:4000"
API_URL = "#{SERVER_URL}/api/links"
API_KEY = "secure_api_key_1"
NUMBER_OF_REQUESTS = 100000
CONTAINER_NAME = "bit"
STATS_FILE = "resource_usage.txt"
class ResourceMonitor
def initialize(@container_name : String)
@running = false
@stats = [] of {timestamp: Time, cpu: Float64, memory: Float64}
end
def start
@running = true
@stats.clear
# Initialize stats file with header
File.write(STATS_FILE, "Timestamp\tCPU(%)\tMemory(MiB)\n")
spawn do
while @running
if stat = capture_stats
# Append each measurement directly to the file
File.open(STATS_FILE, "a") do |file|
file.puts "#{stat[:timestamp].to_unix}\t#{stat[:cpu]}\t#{stat[:memory]}"
end
@stats << stat
end
end
end
end
def stop
@running = false
end
def avg_stats
return {cpu: 0.0, memory: 0.0} if @stats.empty?
total_cpu = 0.0
total_memory = 0.0
@stats.each do |stat|
total_cpu += stat[:cpu]
total_memory += stat[:memory]
end
{
cpu: total_cpu / @stats.size,
memory: total_memory / @stats.size
}
end
private def capture_stats
output = IO::Memory.new
process = Process.run(
"docker", ["stats", "--no-stream", "--format", "{{.CPUPerc}},{{.MemUsage}}", @container_name],
output: output
)
if process.success?
line = output.to_s.strip
parts = line.split(",")
if parts.size == 2
cpu_part = parts[0].gsub("%", "").to_f
# Extract the memory value properly by removing the "MiB" suffix
mem_string = parts[1].split.first
mem_part = mem_string.gsub(/[A-Za-z]+$/, "").to_f
return {timestamp: Time.utc, cpu: cpu_part, memory: mem_part}
end
end
nil
end
end
def check_dependencies
{"docker", "jq", "bombardier"}.each do |cmd|
process = Process.run("which", [cmd], output: Process::Redirect::Close)
unless process.success?
puts "Error: #{cmd} is not installed. Please install it to proceed."
exit(1)
end
end
end
def setup_containers
puts "Setting up..."
process = Process.run("docker", ["compose", "up", "-d"])
unless process.success?
puts "Failed to start Docker containers."
exit(1)
end
puts "Waiting for the application to be ready..."
until begin
HTTP::Client.get("#{SERVER_URL}/api/ping").success?
rescue
false
end
sleep 1.seconds
end
puts "Seeding the database..."
process = Process.run("docker", ["compose", "exec", "app", "sh", "-c", "sqlite3 ./sqlite/data.db < ./db/seed.sql"])
unless process.success?
puts "Error on seeding database"
exit(1)
end
puts "Checking seed results..."
until begin
HTTP::Client.get(
"#{API_URL}?limit=1",
headers: HTTP::Headers{"X-Api-Key" => API_KEY}
).success?
rescue
false
end
sleep 2.seconds
end
end
def run_benchmark
puts "Fetching all created links from /api/links..."
response = HTTP::Client.get(
"#{API_URL}?limit=10000",
headers: HTTP::Headers{"X-Api-Key" => API_KEY}
)
sleep 2.seconds
unless response.success?
puts "Failed to fetch links. Status: #{response.status_code}"
exit(1)
end
data = JSON.parse(response.body)
links = data["data"].as_a.map { |link| link["refer"].as_s }
random_link = links.sample
puts "Selected link for benchmarking: #{random_link}"
puts "Starting benchmark with Bombardier..."
sleep 2.seconds
process = Process.new(
"bombardier",
["-n", NUMBER_OF_REQUESTS.to_s, "-l", "--disableKeepAlives", random_link],
output: Process::Redirect::Inherit,
error: Process::Redirect::Inherit
)
status = process.wait
if status.success?
puts "Benchmark completed successfully."
else
puts "Bombardier failed with error code: #{status.exit_code}"
exit(1)
end
end
def analyze_resource_usage
puts "Analyzing resource usage..."
sleep 2.seconds
if File.exists?(STATS_FILE)
lines = File.read_lines(STATS_FILE)
# Skip header
lines = lines[1..-1] if lines.size > 0
if lines.size > 0
total_cpu = 0.0
total_memory = 0.0
peak_cpu = 0.0
peak_memory = 0.0
lines.each do |line|
fields = line.split("\t")
if fields.size >= 3
begin
cpu = fields[1].to_f
memory = fields[2].to_f
total_cpu += cpu
total_memory += memory
# Track peaks in a single pass
peak_cpu = cpu if cpu > peak_cpu
peak_memory = memory if memory > peak_memory
rescue
# Skip invalid lines
end
end
end
avg_cpu = total_cpu / lines.size
avg_memory = total_memory / lines.size
stats_summary = <<-STATS
**** Resource Usage Statistics ****
Measurements: #{lines.size}
Average CPU Usage: #{avg_cpu.round(2)}%
Average Memory Usage: #{avg_memory.round(2)} MiB
Peak CPU Usage: #{peak_cpu.round(2)}%
Peak Memory Usage: #{peak_memory.round(2)} MiB
STATS
File.open(STATS_FILE, "a") do |file|
file.puts "\n" + stats_summary
end
puts File.read(STATS_FILE)
else
puts "No resource usage data collected."
end
else
puts "Resource usage file not found."
end
end
def cleanup
Process.run("docker", ["compose", "down"])
puts "Cleanup completed. Resource usage data saved in #{STATS_FILE}"
end
def main
check_dependencies
setup_containers
monitor = ResourceMonitor.new(CONTAINER_NAME)
monitor.start
begin
run_benchmark
monitor.stop
analyze_resource_usage
ensure
cleanup
end
end
main
docs/API.md
-149
View File
@@ -1,149 +0,0 @@
# API Reference
1. **Ping the API**
- Endpoint: `GET /api/ping`
- Payload: None
- Response: 200
- Response Example
```json
{
"data": "pong"
}
```
2. **Redirect by Slug**
- Endpoint: `GET /:slug`
- Payload: None
- Response: 301
3. **List All Links**
- Endpoint: `GET /api/links`
- Headers: `X-Api-Key`
- Query Parameters:
- `limit` (optional): Number of results per page (default: 100)
- `cursor` (optional): Pagination cursor from previous response
- Response: 200
- Response Example
```json
{
"data": [
{
"id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db",
"refer": "http://localhost:4000/3wP4BQ",
"origin": "https://monocuco.donado.co"
}
],
"pagination": {
"has_more": true,
"next": "75e0a7f4-9c5e-1235-b546-eb9c5e40f7ac"
}
}
```
4. **List link by ID**
- Endpoint: `GET /api/links/:id`
- Headers: `X-Api-Key`
- Payload: None
- Note: This endpoint returns up to 100 of the most recent clicks. For complete click history, use the `/api/links/:id/clicks` endpoint with pagination.
- Response: 200
- Response Example
```json
{
"data": {
"id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db",
"refer": "http://localhost:4000/3wP4BQ",
"origin": "https://monocuco.donado.co",
"clicks": [
{
"id": "730e2202-58f9-478c-a24c-f1c561df6716",
"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0",
"country": "DE",
"browser": "Firefox",
"os": "Mac OS X",
"referer": "Direct",
"created_at": "2024-07-12T19:25:22Z"
}
]
}
}
```
5. **List Clicks for a Link**
- Endpoint: `GET /api/links/:id/clicks`
- Headers: `X-Api-Key`
- Query Parameters:
- `limit` (optional): Number of results per page (default: 100)
- `cursor` (optional): Pagination cursor from previous response
- Response: 200
- Response Example
```json
{
"data": [
{
"id": "730e2202-58f9-478c-a24c-f1c561df6716",
"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0",
"country": "DE",
"browser": "Firefox",
"os": "Mac OS X",
"referer": "Direct",
"created_at": "2024-07-12T19:25:22Z"
}
],
"pagination": {
"has_more": true,
"next": "629e3301-47f8-389b-b24c-f1c561df9825"
}
}
```
6. **Create new link**
- Endpoint: `POST /api/links`
- Payload:
```json
{
"url": "https://example.com"
}
```
- Headers: `X-Api-Key`
- Response: 201
- Response Example:
```json
{
"data": {
"id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db",
"refer": "http://localhost:4000/3wP4BQ",
"origin": "https://example.com",
"clicks": []
}
}
```
7. **Update an existing link by ID**
- Endpoint: `PUT /api/links/:id`
- Payload:
```json
{
"url": "https://newexample.com"
}
```
- Headers: `X-Api-Key`
- Response: 200
- Response Example:
```json
{
"data": {
"id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db",
"refer": "http://localhost:4000/3wP4BQ",
"origin": "https://newexample.com",
"clicks": []
}
}
```
8. **Delete a link by ID**
- Endpoint: `DELETE /api/links/:id`
- Payload: None
- Headers: `X-Api-Key`
- Response: 204
docs/SETUP.md
+126 -45
View File
@@ -73,7 +73,7 @@ Recommended for lower latency communication (no host network traversal)
## Local Development ## Local Development
### Requirements ### Requirements
- Crystal 1.12+ - Crystal 1.18+
- Shards package manager - Shards package manager
- SQLite3 - SQLite3
@@ -109,59 +109,140 @@ ENV=test crystal spec
## Benchmark ## Benchmark
- Colima: cpu 1, mem 1 ### Run
- SoC: Apple M3 Pro
``` ```
~/p/bit> colima start --cpu 1 --memory 1 shards build --release --no-debug --progress --stats
INFO[0000] starting colima shards run benchmark
INFO[0000] runtime: docker ```
INFO[0001] starting ... context=vm
INFO[0076] provisioning ... context=docker ### Output
INFO[0077] starting ... context=docker
INFO[0077] done Chip: Apple M4 Pro. Memory: 24GB
~/p/bit> ./benchmark.cr
Setting up... ```
Waiting for the application to be ready... 1762075350 ~/p/bit> shards build --release --no-debug --progress --stats
Seeding the database... shards run benchmark
Checking seed results... Dependencies are satisfied
Fetching all created links from /api/links... Building: bit
Selected link for benchmarking: http://localhost:4000/slug2576 Parse: 00:00:00.000652375 ( 1.17MB)
Starting benchmark with Bombardier... Semantic (top level): 00:00:00.419246250 ( 163.45MB)
Bombarding http://localhost:4000/slug2576 with 100000 request(s) using 125 connection(s) Semantic (new): 00:00:00.001636125 ( 163.45MB)
100000 / 100000 [===================================================================================================================================================================] 100.00% 7795/s 12s Semantic (type declarations): 00:00:00.019569792 ( 179.45MB)
Semantic (abstract def check): 00:00:00.009145125 ( 195.45MB)
Semantic (restrictions augmenter): 00:00:00.008421709 ( 195.45MB)
Semantic (ivars initializers): 00:00:00.019696584 ( 211.45MB)
Semantic (cvars initializers): 00:00:00.106829666 ( 211.50MB)
Semantic (main): 00:00:00.649298375 ( 499.88MB)
Semantic (cleanup): 00:00:00.000765250 ( 499.88MB)
Semantic (recursive struct check): 00:00:00.000752250 ( 499.88MB)
Codegen (crystal): 00:00:00.521307417 ( 532.38MB)
Codegen (bc+obj): 00:00:00.143842542 ( 532.38MB)
Codegen (linking): 00:00:00.236228750 ( 532.38MB)
Macro runs:
- /opt/homebrew/Cellar/crystal/1.18.2/share/crystal/src/ecr/process.cr: reused previous compilation (00:00:00.003593375)
Codegen (bc+obj):
- all previous .o files were reused
Building: cli
Parse: 00:00:00.000053291 ( 1.17MB)
Semantic (top level): 00:00:00.323534167 ( 163.45MB)
Semantic (new): 00:00:00.001705500 ( 163.45MB)
Semantic (type declarations): 00:00:00.018311958 ( 179.45MB)
Semantic (abstract def check): 00:00:00.007766750 ( 195.45MB)
Semantic (restrictions augmenter): 00:00:00.005686667 ( 195.45MB)
Semantic (ivars initializers): 00:00:00.011239792 ( 211.45MB)
Semantic (cvars initializers): 00:00:00.100870833 ( 211.50MB)
Semantic (main): 00:00:00.285426750 ( 371.62MB)
Semantic (cleanup): 00:00:00.000369875 ( 371.62MB)
Semantic (recursive struct check): 00:00:00.000570917 ( 371.62MB)
Codegen (crystal): 00:00:00.317534875 ( 387.88MB)
Codegen (bc+obj): 00:00:00.097321417 ( 387.88MB)
Codegen (linking): 00:00:00.095931000 ( 387.88MB)
Codegen (bc+obj):
- all previous .o files were reused
Building: benchmark
Parse: 00:00:00.000228500 ( 1.17MB)
Semantic (top level): 00:00:00.242174458 ( 147.78MB)
Semantic (new): 00:00:00.000863333 ( 147.78MB)
Semantic (type declarations): 00:00:00.011527792 ( 147.78MB)
Semantic (abstract def check): 00:00:00.031242333 ( 147.78MB)
Semantic (restrictions augmenter): 00:00:00.003593583 ( 147.78MB)
Semantic (ivars initializers): 00:00:00.006753667 ( 147.78MB)
Semantic (cvars initializers): 00:00:00.028373834 ( 195.78MB)
Semantic (main): 00:00:00.152039542 ( 243.83MB)
Semantic (cleanup): 00:00:00.000249084 ( 243.83MB)
Semantic (recursive struct check): 00:00:00.000460417 ( 243.83MB)
Codegen (crystal): 00:00:00.075461000 ( 259.83MB)
Codegen (bc+obj): 00:00:04.834914333 ( 259.83MB)
Codegen (linking): 00:00:00.119920416 ( 259.83MB)
Codegen (bc+obj):
- no previous .o files were reused
Dependencies are satisfied
Building: benchmark
Executing: benchmark
Cleaning up benchmark database...
Deleted existing database: ./sqlite/data.benchmark.db
Database cleanup completed.
Running database migrations...
Migrating db, current version: 0, target: 20250319192003
OK 20240512214223_create_links.sql
OK 20240512225208_add_slug_index_to_links.sql
OK 20240513115731_create_users.sql
OK 20240513130054_add_api_key_index_to_users.sql
OK 20240711224103_create_clicks.sql
OK 20240714215409_update_slug_size_links.sql
OK 20250316102350_add_country_to_clicks.sql
OK 20250316111734_replace_unkwown_with_null.sql
OK 20250318072657_replace_slug_index_with_covering_index.sql
OK 20250319192003_convert_all_tables_text_ids_to_integer.sql
Migrations completed successfully.
Seeding benchmark database...
Database seeded successfully.
Starting application: ./bit...
Application output will be saved to: app_output.log
Application started with PID: 11638
Using database: ./sqlite/data.benchmark.db
Checking if server is ready at http://localhost:4001...
.Server is ready!
Fetching links from API...
Selected link: http://localhost:4001/slug9391
Starting benchmark with 100000 requests...
Bombarding http://localhost:4001/slug9391 with 100000 request(s) using 125 connection(s)
100000 / 100000 [============================================================================] 100.00% 11078/s 9s
Done! Done!
Statistics Avg Stdev Max Statistics Avg Stdev Max
Reqs/sec 7900.70 7570.15 29263.59 Reqs/sec 11427.28 8889.68 30270.91
Latency 15.89ms 10.22ms 67.32ms Latency 11.02ms 6.55ms 53.91ms
Latency Distribution Latency Distribution
50% 4.82ms 50% 1.85ms
75% 9.24ms 75% 5.37ms
90% 51.61ms 90% 39.36ms
95% 52.74ms 95% 39.87ms
99% 55.07ms 99% 42.66ms
HTTP codes: HTTP codes:
1xx - 0, 2xx - 0, 3xx - 100000, 4xx - 0, 5xx - 0 1xx - 0, 2xx - 0, 3xx - 100000, 4xx - 0, 5xx - 0
others - 0 others - 0
Throughput: 2.14MB/s Throughput: 3.08MB/s
Benchmark completed successfully. Benchmark completed successfully.
Analyzing resource usage...
Timestamp CPU(%) Memory(MiB)
1742763202 0.01 44.83
1742763204 0.01 44.78
1742763206 91.53 68.23
1742763208 92.03 68.17
1742763210 91.0 68.09
1742763212 92.73 68.38
1742763214 92.17 67.66
1742763216 91.1 67.69
1742763218 2.93 67.04
**** Resource Usage Statistics **** **** Resource Usage Statistics ****
Measurements: 9 Measurements: 12
Average CPU Usage: 61.5% Average CPU Usage: 71.5%
Average Memory Usage: 62.76 MiB Average Memory Usage: 39.8 MiB
Peak CPU Usage: 92.73% Peak CPU Usage: 100.0%
Peak Memory Usage: 68.38 MiB Peak Memory Usage: 53.41 MiB
Cleanup completed. Resource usage data saved in resource_usage.txt
**** Files Generated ****
Resource stats: resource_usage.log
Application log: app_output.log
Database: ./sqlite/data.benchmark.db
Stopping application...
Application stopped.
``` ```
docs/openapi.yaml
+496
View File
@@ -0,0 +1,496 @@
openapi: 3.1.0
info:
title: Bit - URL Shortener API
description: A high-performance URL shortener service with click tracking and analytics
version: 1.0.0
contact:
name: API Support
url: https://github.com/sjdonado/bit
servers:
- url: http://localhost:4000
description: Development server
- url: http://localhost:4001
description: Benchmark server
security:
- ApiKeyAuth: []
paths:
/api/ping:
get:
summary: Ping the API
description: Health check endpoint to verify the API is running
operationId: ping
tags:
- Health
security: []
responses:
'200':
description: API is healthy
content:
application/json:
schema:
type: object
properties:
data:
type: string
example: pong
/{slug}:
get:
summary: Redirect by slug
description: Redirects to the original URL and tracks the click asynchronously
operationId: redirectBySlug
tags:
- Redirects
security: []
parameters:
- name: slug
in: path
required: true
description: The short URL slug
schema:
type: string
example: 3wP4BQ
- name: utm_source
in: query
required: false
description: UTM source parameter for tracking
schema:
type: string
example: email_campaign
responses:
'301':
description: Redirect to original URL
headers:
Location:
description: The original URL
schema:
type: string
example: https://example.com
X-Forwarded-For:
description: Client IP address
schema:
type: string
User-Agent:
description: User agent string
schema:
type: string
'404':
description: Link not found
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
/api/links:
get:
summary: List all links
description: Retrieve all links for the authenticated user with pagination support
operationId: listLinks
tags:
- Links
parameters:
- name: limit
in: query
description: Number of results per page
schema:
type: integer
default: 100
minimum: 1
maximum: 1000
- name: cursor
in: query
description: Pagination cursor from previous response
schema:
type: string
responses:
'200':
description: List of links
content:
application/json:
schema:
type: object
properties:
data:
type: array
items:
$ref: '#/components/schemas/LinkSummary'
pagination:
$ref: '#/components/schemas/Pagination'
'401':
description: Unauthorized
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
post:
summary: Create new link
description: Create a new shortened link
operationId: createLink
tags:
- Links
requestBody:
required: true
content:
application/json:
schema:
type: object
required:
- url
properties:
url:
type: string
format: uri
description: The URL to shorten
example: https://example.com
responses:
'201':
description: Link created successfully
content:
application/json:
schema:
type: object
properties:
data:
$ref: '#/components/schemas/Link'
'400':
description: Bad request - invalid URL or missing field
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
examples:
missingField:
value:
error: "url: Required field"
invalidUrl:
value:
errors:
url:
- is invalid
'401':
description: Unauthorized
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
/api/links/{id}:
get:
summary: Get link by ID
description: Retrieve a specific link with up to 100 most recent clicks. For complete click history, use /api/links/{id}/clicks
operationId: getLink
tags:
- Links
parameters:
- name: id
in: path
required: true
description: Link ID
schema:
type: integer
format: int64
responses:
'200':
description: Link details
content:
application/json:
schema:
type: object
properties:
data:
$ref: '#/components/schemas/Link'
'404':
description: Link not found
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
'401':
description: Unauthorized
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
put:
summary: Update link
description: Update the URL of an existing link
operationId: updateLink
tags:
- Links
parameters:
- name: id
in: path
required: true
description: Link ID
schema:
type: integer
format: int64
requestBody:
required: true
content:
application/json:
schema:
type: object
required:
- url
properties:
url:
type: string
format: uri
description: The new URL
example: https://newexample.com
responses:
'200':
description: Link updated successfully
content:
application/json:
schema:
type: object
properties:
data:
$ref: '#/components/schemas/Link'
'400':
description: Bad request
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
'401':
description: Unauthorized
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
'403':
description: Forbidden - link belongs to another user
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
'404':
description: Link not found
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
delete:
summary: Delete link
description: Delete a link and all its associated clicks
operationId: deleteLink
tags:
- Links
parameters:
- name: id
in: path
required: true
description: Link ID
schema:
type: integer
format: int64
responses:
'204':
description: Link deleted successfully
'401':
description: Unauthorized
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
'403':
description: Forbidden - link belongs to another user
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
'404':
description: Link not found
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
/api/links/{id}/clicks:
get:
summary: List clicks for a link
description: Retrieve all clicks for a specific link with pagination support
operationId: listClicks
tags:
- Clicks
parameters:
- name: id
in: path
required: true
description: Link ID
schema:
type: integer
format: int64
- name: limit
in: query
description: Number of results per page
schema:
type: integer
default: 100
minimum: 1
maximum: 1000
- name: cursor
in: query
description: Pagination cursor from previous response
schema:
type: string
responses:
'200':
description: List of clicks
content:
application/json:
schema:
type: object
properties:
data:
type: array
items:
$ref: '#/components/schemas/Click'
pagination:
$ref: '#/components/schemas/Pagination'
'401':
description: Unauthorized
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
'404':
description: Link not found
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
components:
securitySchemes:
ApiKeyAuth:
type: apiKey
in: header
name: X-Api-Key
description: API key for authentication
schemas:
LinkSummary:
type: object
properties:
id:
type: integer
format: int64
description: Unique link identifier
example: 1
refer:
type: string
format: uri
description: The shortened URL
example: http://localhost:4000/3wP4BQ
origin:
type: string
format: uri
description: The original URL
example: https://monocuco.donado.co
Link:
allOf:
- $ref: '#/components/schemas/LinkSummary'
- type: object
properties:
clicks:
type: array
description: Array of click records (up to 100 most recent)
items:
$ref: '#/components/schemas/Click'
Click:
type: object
properties:
id:
type: integer
format: int64
description: Unique click identifier
example: 1
user_agent:
type: string
description: User agent string
example: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0
country:
type: string
nullable: true
description: Country code (ISO 3166-1 alpha-2)
example: US
browser:
type: string
nullable: true
description: Browser name
example: Firefox
os:
type: string
nullable: true
description: Operating system
example: Mac OS X
referer:
type: string
nullable: true
description: Referer domain or utm_source
example: Direct
created_at:
type: string
format: date-time
description: Click timestamp
example: 2024-07-12T19:25:22Z
Pagination:
type: object
properties:
has_more:
type: boolean
description: Whether there are more results
example: true
next:
type: integer
format: int64
nullable: true
description: Cursor for next page (link/click ID)
example: 12
Error:
type: object
properties:
error:
type: string
description: Error message
example: Resource not found
required:
- error
ValidationErrors:
type: object
properties:
errors:
type: object
additionalProperties:
type: array
items:
type: string
description: Field-level validation errors
example:
url:
- is invalid
tags:
- name: Health
description: Health check endpoints
- name: Redirects
description: URL redirection and click tracking
- name: Links
description: Link management operations
- name: Clicks
description: Click analytics and tracking
scripts/benchmark.cr
+341
View File
@@ -0,0 +1,341 @@
#!/usr/bin/env crystal
require "http/client"
require "json"
PORT = "4001"
APP_URL = "http://localhost:#{PORT}"
API_URL = "#{APP_URL}/api/links"
API_KEY = "secure_api_key_1"
NUMBER_OF_REQUESTS = 100000
APP_COMMAND = "./bit"
APP_ARGS = [] of String
STATS_FILE = "resource_usage.log"
APP_LOG_FILE = "app_output.log"
DATABASE_URL = "sqlite3://./sqlite/data.benchmark.db?journal_mode=wal&synchronous=normal&foreign_keys=true"
DATABASE_FILE = "./sqlite/data.benchmark.db"
class ResourceMonitor
def initialize(@pid : Int32)
@running = false
@stats = [] of {timestamp: Time, cpu: Float64, memory: Float64}
end
def start
@running = true
@stats.clear
File.write(STATS_FILE, "Timestamp\tCPU(%)\tMemory(MiB)\n")
spawn do
while @running
if stat = capture_stats
File.open(STATS_FILE, "a") do |file|
file.puts "#{stat[:timestamp].to_unix}\t#{stat[:cpu]}\t#{stat[:memory]}"
end
@stats << stat
end
sleep 1.seconds
end
end
end
def stop
@running = false
sleep 1.seconds
end
private def capture_stats
output = IO::Memory.new
process = Process.run(
"ps", ["-p", @pid.to_s, "-o", "%cpu,%mem,rss"],
output: output
)
if process.success?
lines = output.to_s.strip.split("\n")
if lines.size >= 2
data_line = lines[1].strip.split
if data_line.size >= 3
cpu = data_line[0].to_f
# RSS is in KB on macOS, convert to MiB
memory_kb = data_line[2].to_f
memory_mib = memory_kb / 1024.0
return {timestamp: Time.utc, cpu: cpu, memory: memory_mib}
end
end
end
nil
end
def print_summary
return if @stats.empty?
total_cpu = 0.0
total_memory = 0.0
peak_cpu = 0.0
peak_memory = 0.0
@stats.each do |stat|
total_cpu += stat[:cpu]
total_memory += stat[:memory]
peak_cpu = stat[:cpu] if stat[:cpu] > peak_cpu
peak_memory = stat[:memory] if stat[:memory] > peak_memory
end
avg_cpu = total_cpu / @stats.size
avg_memory = total_memory / @stats.size
summary = <<-STATS
**** Resource Usage Statistics ****
Measurements: #{@stats.size}
Average CPU Usage: #{avg_cpu.round(2)}%
Average Memory Usage: #{avg_memory.round(2)} MiB
Peak CPU Usage: #{peak_cpu.round(2)}%
Peak Memory Usage: #{peak_memory.round(2)} MiB
STATS
File.open(STATS_FILE, "a") do |file|
file.puts summary
end
puts summary
end
end
def start_application : Process
puts "Starting application: #{APP_COMMAND}..."
puts "Application output will be saved to: #{APP_LOG_FILE}"
log_file = File.open(APP_LOG_FILE, "w")
process = Process.new(
APP_COMMAND,
APP_ARGS,
output: log_file,
error: log_file,
env: {
"DATABASE_URL" => DATABASE_URL,
"APP_URL" => APP_URL,
"PORT" => PORT,
}
)
puts "Application started with PID: #{process.pid}"
puts "Using database: #{DATABASE_FILE}"
process
end
def stop_application(process : Process)
puts "\nStopping application..."
process.signal(Signal::TERM)
# Give it a few seconds to shut down gracefully
sleep 3.seconds
# Force kill if still running
begin
process.signal(Signal::KILL)
rescue
# Process already terminated
end
puts "Application stopped."
end
def check_dependencies
{"bombardier", "sqlite3", "micrate"}.each do |cmd|
process = Process.run("which", [cmd], output: Process::Redirect::Close)
unless process.success?
puts "Error: #{cmd} is not installed. Please install it to proceed."
case cmd
when "bombardier"
puts " brew install bombardier"
when "sqlite3"
puts " brew install sqlite3"
when "micrate"
puts " shards install"
end
exit(1)
end
end
end
def wait_for_server
puts "Checking if server is ready at #{APP_URL}..."
30.times do
begin
if HTTP::Client.get("#{APP_URL}/api/ping").success?
puts "Server is ready!"
return
end
rescue
# Server not ready yet
end
sleep 1.seconds
print "."
end
puts "\nError: Server is not responding. Please start your application first."
exit(1)
end
def run_benchmark
puts "Fetching links from API..."
response = HTTP::Client.get(
"#{API_URL}?limit=10000",
headers: HTTP::Headers{"X-Api-Key" => API_KEY}
)
unless response.success?
puts "Failed to fetch links. Status: #{response.status_code}"
puts "Make sure your server is running and the API key is correct."
exit(1)
end
data = JSON.parse(response.body)
links = data["data"].as_a.map { |link| link["refer"].as_s }
if links.empty?
puts "No links found. Please seed your database first."
exit(1)
end
random_link = links.sample
puts "Selected link: #{random_link}"
puts "\nStarting benchmark with #{NUMBER_OF_REQUESTS} requests..."
sleep 2.seconds
process = Process.new(
"bombardier",
["-n", NUMBER_OF_REQUESTS.to_s, "-l", "--disableKeepAlives", random_link],
output: Process::Redirect::Inherit,
error: Process::Redirect::Inherit
)
status = process.wait
if status.success?
puts "\nBenchmark completed successfully."
else
puts "\nBombardier failed with error code: #{status.exit_code}"
exit(1)
end
end
def cleanup_database
puts "Cleaning up benchmark database..."
if File.exists?(DATABASE_FILE)
File.delete(DATABASE_FILE)
puts "Deleted existing database: #{DATABASE_FILE}"
end
# Also remove WAL and SHM files if they exist
["#{DATABASE_FILE}-wal", "#{DATABASE_FILE}-shm"].each do |file|
if File.exists?(file)
File.delete(file)
puts "Deleted: #{file}"
end
end
# Ensure sqlite directory exists
Dir.mkdir_p("./sqlite")
puts "Database cleanup completed."
end
def run_migrations
puts "Running database migrations..."
process = Process.run("which", ["micrate"], output: Process::Redirect::Close)
unless process.success?
puts "Error: micrate is not installed. Please install it to proceed."
puts " shards install"
exit(1)
end
process = Process.run(
"micrate",
["up"],
env: {"DATABASE_URL" => DATABASE_URL},
output: Process::Redirect::Inherit,
error: Process::Redirect::Inherit
)
if process.success?
puts "Migrations completed successfully."
else
puts "Error: Migrations failed."
exit(1)
end
end
def seed_database
puts "Seeding benchmark database..."
unless File.exists?("./db/seed.sql")
puts "Warning: ./db/seed.sql not found. Skipping database seeding."
return
end
unless File.exists?(DATABASE_FILE)
puts "Warning: #{DATABASE_FILE} not found. Database may not be initialized."
end
process = Process.run(
"sqlite3",
[DATABASE_FILE],
input: File.open("./db/seed.sql"),
output: Process::Redirect::Inherit,
error: Process::Redirect::Inherit
)
if process.success?
puts "Database seeded successfully."
else
puts "Warning: Database seeding failed. Continuing anyway..."
end
end
def main
check_dependencies
# Setup benchmark database
cleanup_database
run_migrations
seed_database
app_process = start_application
begin
wait_for_server
# Give it a moment to settle
sleep 2.seconds
monitor = ResourceMonitor.new(app_process.pid.to_i32)
monitor.start
run_benchmark
monitor.stop
monitor.print_summary
puts "\n**** Files Generated ****"
puts " Resource stats: #{STATS_FILE}"
puts " Application log: #{APP_LOG_FILE}"
puts " Database: #{DATABASE_FILE}"
ensure
# Always stop the application
stop_application(app_process)
end
end
main
scripts/cli.cr
+1 -1
View File
@@ -21,7 +21,7 @@ OptionParser.parse do |parser|
parser.on("--update-parsers", "Download UA regexes and/or GeoLite2 database") do parser.on("--update-parsers", "Download UA regexes and/or GeoLite2 database") do
puts "=== Starting data files update ===" puts "=== Starting data files update ==="
App::Services::Cli.update_uap_regexes App::Services::Cli.update_uap_regexes
App::Services::Cli.download_geolite_db App::Services::Cli.update_geolite_db
puts "=== Data files updated successfully ===" puts "=== Data files updated successfully ==="
exit exit
end end
shard.lock
+4 -4
View File
@@ -2,11 +2,11 @@ version: 2.0
shards: shards:
backtracer: backtracer:
git: https://github.com/sija/backtracer.cr.git git: https://github.com/sija/backtracer.cr.git
version: 1.2.2 version: 1.2.4
crecto: crecto:
git: https://github.com/fridgerator/crecto.git git: https://github.com/fridgerator/crecto.git
version: 0.12.1 version: 0.14.0
db: db:
git: https://github.com/crystal-lang/crystal-db.git git: https://github.com/crystal-lang/crystal-db.git
@@ -18,7 +18,7 @@ shards:
exception_page: exception_page:
git: https://github.com/crystal-loot/exception_page.git git: https://github.com/crystal-loot/exception_page.git
version: 0.4.1 version: 0.5.0
ipaddress: ipaddress:
git: https://github.com/sija/ipaddress.cr.git git: https://github.com/sija/ipaddress.cr.git
@@ -26,7 +26,7 @@ shards:
kemal: kemal:
git: https://github.com/kemalcr/kemal.git git: https://github.com/kemalcr/kemal.git
version: 1.5.0 version: 1.7.3
maxminddb: maxminddb:
git: https://github.com/delef/maxminddb.cr.git git: https://github.com/delef/maxminddb.cr.git
shard.yml
+6 -3
View File
@@ -1,18 +1,21 @@
name: bit name: bit
version: 1.5.3 version: 1.6.0
authors: authors:
- Juan Rodriguez <sjdonado@icloud.com> - Juan Rodriguez Donado <@sjdonado>
targets: targets:
bit: bit:
main: bit.cr main: bit.cr
cli: cli:
main: scripts/cli.cr main: scripts/cli.cr
benchmark:
main: scripts/benchmark.cr
dependencies: dependencies:
kemal: kemal:
github: kemalcr/kemal github: kemalcr/kemal
version: 1.7.3
sqlite3: sqlite3:
github: crystal-lang/crystal-sqlite3 github: crystal-lang/crystal-sqlite3
crecto: crecto:
@@ -29,6 +32,6 @@ development_dependencies:
spec-kemal: spec-kemal:
github: kemalcr/spec-kemal github: kemalcr/spec-kemal
crystal: ">= 1.12.1" crystal: ">= 1.18.2"
license: MIT license: MIT
spec/integration/link_spec.cr
+26 -26
View File
@@ -15,7 +15,7 @@ describe "App::Controllers::Link" do
body: payload.to_json body: payload.to_json
) )
parsed_response = Hash(String, Hash(String, String | Int64 | Array(Hash(String, String)))).from_json(response.body) parsed_response = Hash(String, Hash(String, String | Int64 | Array(Hash(String, String | Int64)))).from_json(response.body)
parsed_response["data"]["origin"].should eq(payload["url"]) parsed_response["data"]["origin"].should eq(payload["url"])
end end
@@ -87,11 +87,10 @@ describe "App::Controllers::Link" do
test_user = create_test_user() test_user = create_test_user()
test_link = create_test_link(test_user, link) test_link = create_test_link(test_user, link)
serialized_link = App::Serializers::Link.new(test_link)
user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0" user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0"
get(serialized_link.refer, headers: HTTP::Headers{ get("/#{test_link.slug}", headers: HTTP::Headers{
"X-Api-Key" => test_user.api_key.to_s, "X-Api-Key" => test_user.api_key.to_s,
"User-Agent" => user_agent "User-Agent" => user_agent
}) })
@@ -106,12 +105,11 @@ describe "App::Controllers::Link" do
test_user = create_test_user() test_user = create_test_user()
test_link = create_test_link(test_user, link) test_link = create_test_link(test_user, link)
serialized_link = App::Serializers::Link.new(test_link)
user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0" user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0"
referer = "https://example.com/page" referer = "https://example.com/page"
get(serialized_link.refer, headers: HTTP::Headers{ get("/#{test_link.slug}", headers: HTTP::Headers{
"User-Agent" => user_agent, "User-Agent" => user_agent,
"Referer" => referer "Referer" => referer
}) })
@@ -121,7 +119,7 @@ describe "App::Controllers::Link" do
response.headers["Location"].should eq(link) response.headers["Location"].should eq(link)
# Verify that the click was recorded # Verify that the click was recorded
updated_test_link = get_test_link(test_link.id) updated_test_link = get_test_link(test_link.id.not_nil!)
updated_test_link.clicks.size.should eq(test_link.clicks.size + 1) updated_test_link.clicks.size.should eq(test_link.clicks.size + 1)
# Verify click details # Verify click details
@@ -137,14 +135,16 @@ describe "App::Controllers::Link" do
test_user = create_test_user() test_user = create_test_user()
test_link = create_test_link(test_user, link) test_link = create_test_link(test_user, link)
serialized_link = App::Serializers::Link.new(test_link)
# Add utm_source parameter # Add utm_source parameter
get("#{serialized_link.refer}?utm_source=email_campaign") user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0"
get("/#{test_link.slug}?utm_source=email_campaign", headers: HTTP::Headers{
"User-Agent" => user_agent
})
Fiber.yield sleep 0.2.seconds # Wait for async click creation
updated_test_link = get_test_link(test_link.id) updated_test_link = get_test_link(test_link.id.not_nil!)
latest_click = updated_test_link.clicks.last latest_click = updated_test_link.clicks.last
latest_click.referer.should eq("email_campaign") latest_click.referer.should eq("email_campaign")
end end
@@ -152,7 +152,7 @@ describe "App::Controllers::Link" do
it "should return 404 - link does not exist" do it "should return 404 - link does not exist" do
test_user = create_test_user() test_user = create_test_user()
get("https://localhost:4001/R4kj2") get("/R4kj2")
expected = {"error" => "Resource not found"}.to_json expected = {"error" => "Resource not found"}.to_json
response.status_code.should eq(404) response.status_code.should eq(404)
@@ -171,7 +171,7 @@ describe "App::Controllers::Link" do
get("/api/links", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) get("/api/links", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body) parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body)
# Check that each link is in the response data # Check that each link is in the response data
origins = parsed_response["data"].as(Array).map { |link| link["origin"] } origins = parsed_response["data"].as(Array).map { |link| link["origin"] }
@@ -191,7 +191,7 @@ describe "App::Controllers::Link" do
get("/api/links?limit=2", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) get("/api/links?limit=2", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body) parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body)
parsed_response["data"].as(Array).size.should eq(2) parsed_response["data"].as(Array).size.should eq(2)
parsed_response["pagination"].as(Hash)["has_more"].should be_true parsed_response["pagination"].as(Hash)["has_more"].should be_true
parsed_response["pagination"].as(Hash)["next"].should_not be_nil parsed_response["pagination"].as(Hash)["next"].should_not be_nil
@@ -206,12 +206,12 @@ describe "App::Controllers::Link" do
# Get first page # Get first page
get("/api/links?limit=2", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) get("/api/links?limit=2", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
first_page = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body) first_page = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body)
cursor = first_page["pagination"].as(Hash)["next"] cursor = first_page["pagination"].as(Hash)["next"]
# Get second page using cursor # Get second page using cursor
get("/api/links?limit=2&cursor=#{cursor}", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) get("/api/links?limit=2&cursor=#{cursor}", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
second_page = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body) second_page = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body)
# Ensure different links are returned # Ensure different links are returned
first_page_ids = first_page["data"].as(Array).map { |link| link["id"] } first_page_ids = first_page["data"].as(Array).map { |link| link["id"] }
@@ -234,7 +234,7 @@ describe "App::Controllers::Link" do
get("/api/links", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) get("/api/links", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body) parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body)
parsed_response["data"].as(Array).size.should eq(3) parsed_response["data"].as(Array).size.should eq(3)
origins = parsed_response["data"].as(Array).map { |link| link["origin"] } origins = parsed_response["data"].as(Array).map { |link| link["origin"] }
@@ -265,7 +265,7 @@ describe "App::Controllers::Link" do
get("/api/links/#{test_link.id}", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) get("/api/links/#{test_link.id}", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
parsed_response = Hash(String, Hash(String, String | Int64 | Array(Hash(String, String)))).from_json(response.body) parsed_response = Hash(String, Hash(String, String | Int64 | Array(Hash(String, String | Int64)))).from_json(response.body)
parsed_response["data"]["origin"].should eq(link) parsed_response["data"]["origin"].should eq(link)
parsed_response["data"]["clicks"].as(Array).size.should eq(100) parsed_response["data"]["clicks"].as(Array).size.should eq(100)
end end
@@ -273,7 +273,7 @@ describe "App::Controllers::Link" do
it "should return 404 - link does not exist" do it "should return 404 - link does not exist" do
test_user = create_test_user() test_user = create_test_user()
get("/api/links/1", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) get("/api/links/999999", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
expected = {"error" => "Resource not found"}.to_json expected = {"error" => "Resource not found"}.to_json
response.status_code.should eq(404) response.status_code.should eq(404)
@@ -301,7 +301,7 @@ describe "App::Controllers::Link" do
get("/api/links/#{test_link.id}/clicks", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) get("/api/links/#{test_link.id}/clicks", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
parsed_response = Hash(String, Array(Hash(String, String)) | Hash(String, Bool | String?)).from_json(response.body) parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body)
parsed_response["data"].as(Array).size.should eq(5) parsed_response["data"].as(Array).size.should eq(5)
parsed_response["pagination"].as(Hash)["has_more"].should be_false parsed_response["pagination"].as(Hash)["has_more"].should be_false
end end
@@ -317,7 +317,7 @@ describe "App::Controllers::Link" do
get("/api/links/#{test_link.id}/clicks?limit=3", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) get("/api/links/#{test_link.id}/clicks?limit=3", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
parsed_response = Hash(String, Array(Hash(String, String)) | Hash(String, Bool | String?)).from_json(response.body) parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body)
parsed_response["data"].as(Array).size.should eq(3) parsed_response["data"].as(Array).size.should eq(3)
parsed_response["pagination"].as(Hash)["has_more"].should be_true parsed_response["pagination"].as(Hash)["has_more"].should be_true
parsed_response["pagination"].as(Hash)["next"].should_not be_nil parsed_response["pagination"].as(Hash)["next"].should_not be_nil
@@ -334,12 +334,12 @@ describe "App::Controllers::Link" do
# Get first page # Get first page
get("/api/links/#{test_link.id}/clicks?limit=3", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) get("/api/links/#{test_link.id}/clicks?limit=3", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
first_page = Hash(String, Array(Hash(String, String)) | Hash(String, Bool | String?)).from_json(response.body) first_page = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body)
cursor = first_page["pagination"].as(Hash)["next"] cursor = first_page["pagination"].as(Hash)["next"]
# Get second page using cursor # Get second page using cursor
get("/api/links/#{test_link.id}/clicks?limit=3&cursor=#{cursor}", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) get("/api/links/#{test_link.id}/clicks?limit=3&cursor=#{cursor}", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
second_page = Hash(String, Array(Hash(String, String)) | Hash(String, Bool | String?)).from_json(response.body) second_page = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body)
# Ensure different clicks are returned # Ensure different clicks are returned
first_page_ids = first_page["data"].as(Array).map { |click| click["id"] } first_page_ids = first_page["data"].as(Array).map { |click| click["id"] }
@@ -352,7 +352,7 @@ describe "App::Controllers::Link" do
it "should return 404 - link does not exist" do it "should return 404 - link does not exist" do
test_user = create_test_user() test_user = create_test_user()
get("/api/links/nonexistent_id/clicks", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) get("/api/links/999999/clicks", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
expected = {"error" => "Resource not found"}.to_json expected = {"error" => "Resource not found"}.to_json
response.status_code.should eq(404) response.status_code.should eq(404)
@@ -381,7 +381,7 @@ describe "App::Controllers::Link" do
body: payload.to_json body: payload.to_json
) )
parsed_response = Hash(String, Hash(String, String | Int64 | Array(Hash(String, String)))).from_json(response.body) parsed_response = Hash(String, Hash(String, String | Int64 | Array(Hash(String, String | Int64)))).from_json(response.body)
parsed_response["data"]["origin"].should eq(payload["url"]) parsed_response["data"]["origin"].should eq(payload["url"])
end end
@@ -390,7 +390,7 @@ describe "App::Controllers::Link" do
payload = {"url" => "https://kagi.com.co"} payload = {"url" => "https://kagi.com.co"}
put( put(
"/api/links/1", "/api/links/999999",
headers: HTTP::Headers{"Content-Type" => "application/json", "X-Api-Key" => test_user.api_key.to_s}, headers: HTTP::Headers{"Content-Type" => "application/json", "X-Api-Key" => test_user.api_key.to_s},
body: payload.to_json body: payload.to_json
) )
@@ -428,7 +428,7 @@ describe "App::Controllers::Link" do
it "should return 404 - link does not exist" do it "should return 404 - link does not exist" do
test_user = create_test_user() test_user = create_test_user()
delete("/api/links/1", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) delete("/api/links/999999", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
expected = {"error" => "Resource not found"}.to_json expected = {"error" => "Resource not found"}.to_json
response.status_code.should eq(404) response.status_code.should eq(404)
spec/spec_helper.cr
+9 -8
View File
@@ -32,12 +32,12 @@ def create_test_user
raise "Test user creation failed #{error_messages}" raise "Test user creation failed #{error_messages}"
end end
user changeset.instance
end end
def create_test_link(user, url) def create_test_link(user, url)
link = App::Models::Link.new link = App::Models::Link.new
link.slug = App::Services::SlugService.shorten_url(url, user.id) link.slug = App::Services::SlugService.shorten_url(url, user.id.not_nil!)
link.url = url link.url = url
link.user = user link.user = user
@@ -47,9 +47,10 @@ def create_test_link(user, url)
raise "Test link creation failed: #{error_messages}" raise "Test link creation failed: #{error_messages}"
end end
link.clicks = [] of App::Models::Click inserted_link = changeset.instance
inserted_link.clicks = [] of App::Models::Click
link inserted_link
end end
def create_test_click(link) def create_test_click(link)
@@ -61,17 +62,17 @@ def create_test_click(link)
click.country = "US" click.country = "US"
click.created_at = Time.utc click.created_at = Time.utc
click.link = link click.link = link
click.link_id = link.id click.link_id = link.id.not_nil!
changeset = App::Lib::Database.insert(click) changeset = App::Lib::Database.insert(click)
unless changeset.valid? unless changeset.valid?
error_messages = changeset.errors.map { |error| "#{error}" }.join(", ") error_messages = changeset.errors.map { |error| "#{error}" }.join(", ")
raise "Test click creation failed: #{error_messages}" raise "Test click creation failed: #{error_messages}"
end end
click changeset.instance
end end
def get_test_link(link_id: Int64) def get_test_link(link_id : Int64)
query = App::Lib::Database::Query.where(id: link_id).limit(1) query = App::Lib::Database::Query.where(id: link_id).limit(1)
link = App::Lib::Database.all(App::Models::Link, query, preload: [:clicks]).first? link = App::Lib::Database.all(App::Models::Link, query, preload: [:clicks]).first?
@@ -80,6 +81,6 @@ def get_test_link(link_id: Int64)
link link
end end
def delete_test_link(link_id: Int64) def delete_test_link(link_id : Int64)
App::Lib::Database.raw_exec("DELETE FROM links WHERE id = (?)", link_id) # tempfix: Database.delete does not work App::Lib::Database.raw_exec("DELETE FROM links WHERE id = (?)", link_id) # tempfix: Database.delete does not work
end end