Compare commits
33 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Juan Rodriguez Donado
|
2c55ebf406 | ||
|
sjdonado
|
2a4264e4c5 | ||
|
Juan Rodriguez Donado
|
adfebe9d63 | ||
|
sjdonado
|
45bf499d21 | ||
|
Juan Rodriguez Donado
|
1552b5ce09 | ||
|
sjdonado
|
81f3c95c2b | ||
|
Juan Rodriguez Donado
|
3776621fe9 | ||
|
sjdonado
|
0d68b0d6e1 | ||
|
Juan Rodriguez Donado
|
8048277f1d | ||
|
sjdonado
|
dcff88f55e | ||
|
Juan Rodriguez Donado
|
f7add0116e | ||
|
sjdonado
|
5f702e69c9 | ||
|
Juan Rodriguez Donado
|
2feeff70bc | ||
|
sjdonado
|
1bb42684c3 | ||
|
sjdonado
|
b6e7c45c80 | ||
|
sjdonado
|
7d275685b4 | ||
|
sjdonado
|
44dab3ca5a | ||
|
Juan Rodriguez Donado
|
b8c1269d6e | ||
|
sjdonado
|
9fcd478f86 | ||
|
sjdonado
|
7c6d67c0c7 | ||
|
Juan Rodriguez Donado
|
353cf68852 | ||
|
sjdonado
|
fef015ce53 | ||
|
sjdonado
|
80a59094f9 | ||
|
sjdonado
|
6f5ad76718 | ||
|
sjdonado
|
52497235b9 | ||
|
sjdonado
|
aec073b696 | ||
|
sjdonado
|
6e587f0176 | ||
|
sjdonado
|
0c89fad713 | ||
|
sjdonado
|
046b15bdce | ||
|
sjdonado
|
d0412d802b | ||
|
sjdonado
|
c9e7ad1d99 | ||
|
sjdonado
|
9fa7142ea7 | ||
|
Juan Rodriguez
|
c539662235 |
+40
-5
@@ -1,9 +1,44 @@
|
||||
.git
|
||||
/bin/
|
||||
/.shards/
|
||||
/spec/
|
||||
/sqlite/
|
||||
.gitignore
|
||||
.github
|
||||
|
||||
/bin/
|
||||
/bit
|
||||
/cli
|
||||
/benchmark
|
||||
*.dwarf
|
||||
*.o
|
||||
*.a
|
||||
|
||||
# Dependencies cache
|
||||
/.shards/
|
||||
/lib/.shards/
|
||||
|
||||
/spec/
|
||||
|
||||
# Database files (should be mounted as volumes)
|
||||
/sqlite/
|
||||
*.db
|
||||
*.db-shm
|
||||
*.db-wal
|
||||
|
||||
# Logs and temporary files
|
||||
*.log
|
||||
|
||||
# Documentation
|
||||
/docs/
|
||||
benchmark.cr
|
||||
*.md
|
||||
README.md
|
||||
CODE_OF_CONDUCT.md
|
||||
CONTRIBUTING.md
|
||||
LICENSE
|
||||
DOCKER_MIGRATION.md
|
||||
|
||||
# Development environment
|
||||
.env*
|
||||
.editorconfig
|
||||
|
||||
# Docker files (not needed inside image)
|
||||
Dockerfile
|
||||
docker-compose.yml
|
||||
.dockerignore
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
name: Deploy API Documentation
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths:
|
||||
- 'docs/openapi.yaml'
|
||||
- '.github/workflows/deploy-docs.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Generate Swagger UI
|
||||
uses: Legion2/swagger-ui-action@v1
|
||||
with:
|
||||
output: swagger-ui
|
||||
spec-file: docs/openapi.yaml
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Deploy to GitHub Pages
|
||||
uses: peaceiris/actions-gh-pages@v3
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
publish_dir: swagger-ui
|
||||
@@ -8,12 +8,9 @@ on:
|
||||
types: [published]
|
||||
|
||||
jobs:
|
||||
build-platforms:
|
||||
name: Build Platforms
|
||||
build-and-push:
|
||||
name: Build and Push Multi-Platform
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
platform: [linux/amd64, linux/arm64]
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
@@ -22,8 +19,11 @@ jobs:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v2
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
@@ -40,45 +40,12 @@ jobs:
|
||||
echo "version=latest" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Build and push platform image
|
||||
- name: Build and push multi-platform image
|
||||
uses: docker/build-push-action@v5
|
||||
env:
|
||||
CRYSTAL_WORKERS: ${{ matrix.platform == 'linux/amd64' && 4 || 2 }}
|
||||
with:
|
||||
context: .
|
||||
platforms: ${{ matrix.platform }}
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: |
|
||||
sjdonado/bit:${{ steps.version.outputs.version }}-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
|
||||
build-args: |
|
||||
TARGETARCH=${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
|
||||
tags: sjdonado/bit:${{ steps.version.outputs.version }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
create-manifest:
|
||||
name: Create Manifest
|
||||
runs-on: ubuntu-latest
|
||||
needs: build-platforms
|
||||
permissions:
|
||||
packages: write
|
||||
steps:
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Determine version
|
||||
id: version
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "release" ]; then
|
||||
echo "version=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "version=latest" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Create manifest
|
||||
run: |
|
||||
docker buildx imagetools create \
|
||||
-t sjdonado/bit:${{ steps.version.outputs.version }} \
|
||||
sjdonado/bit:${{ steps.version.outputs.version }}-{amd64,arm64}
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
name: Update Parsers
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Run every two weeks on Sunday at 00:00 UTC (1st and 3rd Sunday of each month)
|
||||
- cron: '0 0 1-7,15-21 * 0'
|
||||
workflow_dispatch: # Allow manual trigger
|
||||
|
||||
jobs:
|
||||
update-parsers:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install Crystal
|
||||
uses: crystal-lang/install-crystal@v1
|
||||
|
||||
- name: Install dependencies
|
||||
run: shards install
|
||||
|
||||
- name: Build CLI
|
||||
run: crystal build scripts/cli.cr -o cli
|
||||
|
||||
- name: Update parsers
|
||||
run: ./cli --update-parsers
|
||||
|
||||
- name: Check for changes
|
||||
id: changes
|
||||
run: |
|
||||
if [ -n "$(git status --porcelain)" ]; then
|
||||
echo "has_changes=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "has_changes=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Create Pull Request
|
||||
if: steps.changes.outputs.has_changes == 'true'
|
||||
uses: peter-evans/create-pull-request@v6
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
commit-message: 'chore: update parsers'
|
||||
title: 'chore: Update UA regexes and GeoLite2 database'
|
||||
body: |
|
||||
## Automated Parser Update
|
||||
|
||||
This PR updates the following data files:
|
||||
- User Agent parsing regexes
|
||||
- GeoLite2 database
|
||||
|
||||
**Triggered by**: Scheduled workflow (runs every 2 weeks)
|
||||
**Date**: ${{ github.event.repository.updated_at }}
|
||||
|
||||
Please review the changes and merge if everything looks good.
|
||||
branch: automated/update-parsers
|
||||
delete-branch: true
|
||||
labels: |
|
||||
dependencies
|
||||
automated
|
||||
+3
-1
@@ -7,4 +7,6 @@
|
||||
/sqlite/
|
||||
.env.production
|
||||
|
||||
resource_usage.*
|
||||
*.log
|
||||
bit
|
||||
cli
|
||||
|
||||
+18
-22
@@ -1,41 +1,37 @@
|
||||
FROM debian:bookworm-slim AS build
|
||||
FROM alpine:edge AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ENV ENV=production
|
||||
WORKDIR /usr/src/app
|
||||
|
||||
RUN apt-get update && apt-get install -y \
|
||||
curl \
|
||||
gnupg \
|
||||
ca-certificates \
|
||||
&& mkdir -p /etc/apt/keyrings \
|
||||
&& curl -fsSL https://packagecloud.io/84codes/crystal/gpgkey | gpg --dearmor > /etc/apt/trusted.gpg.d/84codes_crystal.gpg \
|
||||
&& echo "deb [signed-by=/etc/apt/trusted.gpg.d/84codes_crystal.gpg] https://packagecloud.io/84codes/crystal/debian/ bookworm main" > /etc/apt/sources.list.d/84codes_crystal.list
|
||||
|
||||
RUN apt-get update && apt-get install -y \
|
||||
RUN apk add --no-cache \
|
||||
crystal \
|
||||
libssl-dev \
|
||||
libyaml-dev \
|
||||
libsqlite3-dev \
|
||||
shards \
|
||||
openssl-dev \
|
||||
yaml-dev \
|
||||
sqlite-dev \
|
||||
libevent-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
tzdata
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN shards install --production
|
||||
RUN shards build --release --no-debug --progress --stats
|
||||
|
||||
FROM debian:bookworm-slim AS runtime
|
||||
FROM alpine:latest AS runtime
|
||||
|
||||
ENV ENV=production
|
||||
WORKDIR /usr/src/app
|
||||
|
||||
RUN apt-get update && apt-get install -y \
|
||||
libssl3 \
|
||||
libyaml-0-2 \
|
||||
sqlite3 \
|
||||
libevent-2.1-7 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
RUN apk add --no-cache \
|
||||
gc-dev \
|
||||
pcre2 \
|
||||
libevent \
|
||||
sqlite-libs \
|
||||
openssl \
|
||||
yaml \
|
||||
gmp \
|
||||
libgcc \
|
||||
tzdata
|
||||
|
||||
RUN mkdir -p sqlite
|
||||
|
||||
|
||||
@@ -1,30 +1,93 @@
|
||||
[](https://hub.docker.com/r/sjdonado/bit)
|
||||
[](https://hub.docker.com/r/sjdonado/bit)
|
||||
[](https://hub.docker.com/r/sjdonado/bit)
|
||||
|
||||
Lightweight URL shortener (API-only) with minimal resource requirements. Avg memory consumption under pressure is around **60MiB**, CPU single core consumption 60%.
|
||||
|
||||
Highly performant: 6K+ reqs/sec, latency 20ms (100000 requests using 125 connections, [benchmark](docs/SETUP.md#benchmark)).
|
||||
|
||||
Self-hosted: [Dokku](docs/SETUP.md#dokku), [Docker Compose](docs/SETUP.md#docker-compose).
|
||||
|
||||
Images available on [Docker Hub](https://hub.docker.com/r/sjdonado/bit/tags).
|
||||
|
||||
## Why bit?
|
||||
It is feature-complete by design: simple and reliable without unnecessary bloat. Bug fixes will continue, but new features aren't planned.
|
||||
## Features
|
||||
|
||||
- Minimal tracking setup: Country, browser, OS, referer. No cookies or persistent tracking mechanisms are used beyond what's available from a basic client's request.
|
||||
- Provides standard `X-Forwarded-For` header support to enable extended capabilities.
|
||||
- Multiple users are supported via API key authentication. Users can create, list and delete keys via the [CLI](docs/SETUP.md#cli).
|
||||
- Includes `X-Forwarded-For` header.
|
||||
- Multiple users are supported via API key authentication. Create, list and delete keys via the [CLI](docs/SETUP.md#cli).
|
||||
- Easy to extend, Ruby on Rails inspired setup.
|
||||
- Auto update UA regexes and GeoLite2 database.
|
||||
|
||||
## Minimum Requirements
|
||||
- 100MB disk space
|
||||
- 70MiB RAM
|
||||
- x86_64 or ARM64
|
||||
## Why bit?
|
||||
|
||||
**Fast:** **11k req/sec**, latency 11ms, 40MiB avg memory usage (100k requests using 125 connections, [benchmark](docs/SETUP.md#benchmark)).
|
||||
|
||||
**Lightweight:** Minimal dependencies, image size under 20 MiB, memory usage under 60 MiB at peak.
|
||||
|
||||
**Self-hosted:** [Dokku](docs/SETUP.md#dokku), [Docker Compose](docs/SETUP.md#docker-compose).
|
||||
|
||||
**Production ready:** Feature-complete by design, simple and reliable without unnecessary bloat. Bug fixes will continue, but new features aren't planned.
|
||||
|
||||
## Run It Anywhere
|
||||
|
||||
All images available on [Docker Hub](https://hub.docker.com/r/sjdonado/bit/tags).
|
||||
|
||||
### Docker
|
||||
|
||||
```bash
|
||||
docker run \
|
||||
--name bit \
|
||||
-p 4000:4000 \
|
||||
-e ENV="production" \
|
||||
-e DATABASE_URL="sqlite3://./sqlite/data.db" \
|
||||
-e APP_URL="http://localhost:4000" \
|
||||
-e ADMIN_NAME="Admin" \
|
||||
-e ADMIN_API_KEY=$(openssl rand -base64 32) \
|
||||
sjdonado/bit
|
||||
|
||||
# Create a new user
|
||||
# docker exec -it bit cli --create-user=Admin
|
||||
```
|
||||
|
||||
### Docker Compose
|
||||
|
||||
```bash
|
||||
docker-compose up
|
||||
|
||||
# Optional: Generate an api key
|
||||
# docker-compose exec -it app cli --create-user=Admin
|
||||
```
|
||||
|
||||
### Dokku
|
||||
|
||||
- Dockerfile
|
||||
```dockerfile
|
||||
FROM sjdonado/bit
|
||||
```
|
||||
|
||||
- Over ssh
|
||||
|
||||
```bash
|
||||
dokku apps:create bit
|
||||
|
||||
dokku domains:set bit bit.yourdomain.com
|
||||
dokku letsencrypt:enable bit
|
||||
|
||||
dokku storage:ensure-directory bit-sqlite
|
||||
dokku storage:mount bit /var/lib/dokku/data/storage/bit-sqlite:/usr/src/app/sqlite/
|
||||
|
||||
dokku config:set bit DATABASE_URL="sqlite3://./sqlite/data.db" APP_URL=https://bit.yourdomain.com ADMIN_NAME=Admin ADMIN_API_KEY=$(openssl rand -base64 32)
|
||||
|
||||
dokku ports:add bit http:80:4000
|
||||
dokku ports:add bit https:443:4000
|
||||
|
||||
# Create a new user
|
||||
# dokku run bit cli --create-user=Admin
|
||||
```
|
||||
|
||||
### Dokku (subnetwork)
|
||||
Recommended for lower latency communication (no host network traversal)
|
||||
|
||||
```bash
|
||||
dokku network:create bit-net
|
||||
dokku network:set bit attach-post-create bit-net
|
||||
dokku network:set myapp attach-post-create bit-net
|
||||
```
|
||||
|
||||
## Documentation
|
||||
- [API Reference](docs/API.md)
|
||||
- [Setup](docs/SETUP.md)
|
||||
- [API Reference](https://sjdonado.github.io/bit/)
|
||||
- [Local Development](docs/SETUP.md)
|
||||
|
||||
## Contributing
|
||||
Found an issue or have a suggestion? Please follow our [contribution guidelines](CONTRIBUTING.md).
|
||||
|
||||
+22
-87
@@ -4,82 +4,6 @@ module App::Controllers
|
||||
include App::Lib
|
||||
include App::Services
|
||||
|
||||
# Buffered channel to hold click data
|
||||
@@click_channel = Channel(NamedTuple(
|
||||
link_id: Int64,
|
||||
remote_address: String,
|
||||
user_agent: String?,
|
||||
referer: String
|
||||
)).new(1024)
|
||||
|
||||
@@processor_started = begin
|
||||
spawn do
|
||||
batch_size = 64
|
||||
batch = [] of NamedTuple(
|
||||
link_id: Int64,
|
||||
remote_address: String,
|
||||
user_agent: String?,
|
||||
referer: String
|
||||
)
|
||||
|
||||
loop do
|
||||
select
|
||||
when click_data = @@click_channel.receive
|
||||
batch << click_data
|
||||
|
||||
# Collect clicks until we have a batch or a timeout
|
||||
if batch.size >= batch_size
|
||||
process_click_batch(batch)
|
||||
batch.clear
|
||||
end
|
||||
when timeout(0.5.seconds)
|
||||
# Process whatever we have after timeout
|
||||
unless batch.empty?
|
||||
process_click_batch(batch)
|
||||
batch.clear
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
true
|
||||
end
|
||||
|
||||
private def self.process_click_batch(batch)
|
||||
clicks = [] of App::Models::Click
|
||||
|
||||
batch.each do |click_data|
|
||||
begin
|
||||
client_ip = IpLookup.ip_from_address(click_data[:remote_address])
|
||||
family, _, _, os = UserAgent.parse(click_data[:user_agent] || "")
|
||||
|
||||
click = App::Models::Click.new
|
||||
click.link_id = click_data[:link_id]
|
||||
click.country = client_ip ? IpLookup.country(client_ip) : nil
|
||||
click.user_agent = click_data[:user_agent]
|
||||
click.browser = family
|
||||
click.os = os.try &.[0] # OS family
|
||||
click.referer = click_data[:referer]
|
||||
|
||||
clicks << click
|
||||
rescue ex
|
||||
Log.error { "Click data processing error: #{ex.message}" }
|
||||
end
|
||||
end
|
||||
|
||||
# Batch insert clicks if any were successfully processed
|
||||
unless clicks.empty?
|
||||
begin
|
||||
multi = Crecto::Multi.new
|
||||
clicks.each do |click|
|
||||
multi.insert(click)
|
||||
end
|
||||
Database.transaction(multi)
|
||||
rescue ex
|
||||
Log.error { "Batch click insertion error: #{ex.message}" }
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def self.redirect_handler
|
||||
->(env : HTTP::Server::Context) {
|
||||
link_id, url = Database.raw_query("SELECT id, url FROM links WHERE slug = (?) LIMIT 1", env.params.url["slug"]) do |result|
|
||||
@@ -88,21 +12,32 @@ module App::Controllers
|
||||
|
||||
remote_address = env.request.headers["Cf-Connecting-Ip"]? || env.request.remote_address.to_s
|
||||
|
||||
# Send redirect immediately
|
||||
env.response.status_code = 301
|
||||
env.response.headers.add("Location", url)
|
||||
env.response.headers.add("X-Forwarded-For", remote_address)
|
||||
if user_agent = env.request.headers["User-Agent"]?
|
||||
env.response.headers.add("User-Agent", user_agent)
|
||||
end
|
||||
|
||||
begin
|
||||
@@click_channel.send({
|
||||
link_id: link_id,
|
||||
remote_address: remote_address,
|
||||
user_agent: env.request.headers["User-Agent"]?,
|
||||
referer: env.request.headers["Referer"]?.try { |r| URI.parse(r).host rescue r } || env.params.query["utm_source"]? || "Direct"
|
||||
})
|
||||
rescue Channel::ClosedError
|
||||
Log.error { "Click channel closed" }
|
||||
rescue ex
|
||||
Log.error { "Error queuing click: #{ex.message}" }
|
||||
# non-blocking click proccessing
|
||||
spawn do
|
||||
begin
|
||||
client_ip = IpLookup.ip_from_address(remote_address)
|
||||
family, _, _, os = UserAgent.parse(env.request.headers["User-Agent"]? || "")
|
||||
|
||||
click = App::Models::Click.new
|
||||
click.link_id = link_id
|
||||
click.country = client_ip ? IpLookup.country(client_ip) : nil
|
||||
click.user_agent = env.request.headers["User-Agent"]?
|
||||
click.browser = family
|
||||
click.os = os.try &.[0]
|
||||
click.referer = env.request.headers["Referer"]?.try { |r| URI.parse(r).host rescue r } || env.params.query["utm_source"]? || "Direct"
|
||||
|
||||
Database.insert(click)
|
||||
rescue ex
|
||||
Log.error { "Click tracking error: #{ex.message}" }
|
||||
end
|
||||
end
|
||||
}
|
||||
end
|
||||
|
||||
@@ -145,7 +145,7 @@ module App::Controllers
|
||||
"data" => items.map { |item| yield item },
|
||||
"pagination" => {
|
||||
"has_more" => has_more,
|
||||
"next_cursor" => next_cursor
|
||||
"next" => next_cursor
|
||||
}
|
||||
})
|
||||
end
|
||||
|
||||
@@ -23,7 +23,7 @@ module App::Lib
|
||||
missing_fields = required_fields.reject { |field| json_params.has_key?(field) }
|
||||
|
||||
unless missing_fields.empty?
|
||||
error_message = missing_fields.join(", ") + " required"
|
||||
error_message = "#{missing_fields.first}: Required field"
|
||||
raise App::BadRequestException.new(@env, error_message)
|
||||
end
|
||||
|
||||
|
||||
+1
-1
@@ -73,7 +73,7 @@ module App::Services::Cli
|
||||
end
|
||||
end
|
||||
|
||||
def self.download_geolite_db
|
||||
def self.update_geolite_db
|
||||
puts "Downloading GeoLite2 Country database..."
|
||||
|
||||
FileUtils.mkdir_p("data")
|
||||
|
||||
-258
@@ -1,258 +0,0 @@
|
||||
#!/usr/bin/env crystal
|
||||
|
||||
require "http/client"
|
||||
require "json"
|
||||
require "file_utils"
|
||||
|
||||
# Configuration variables
|
||||
SERVER_URL = "http://localhost:4000"
|
||||
API_URL = "#{SERVER_URL}/api/links"
|
||||
API_KEY = "secure_api_key_1"
|
||||
NUMBER_OF_REQUESTS = 100000
|
||||
|
||||
CONTAINER_NAME = "bit"
|
||||
STATS_FILE = "resource_usage.txt"
|
||||
|
||||
class ResourceMonitor
|
||||
def initialize(@container_name : String)
|
||||
@running = false
|
||||
@stats = [] of {timestamp: Time, cpu: Float64, memory: Float64}
|
||||
end
|
||||
|
||||
def start
|
||||
@running = true
|
||||
@stats.clear
|
||||
|
||||
# Initialize stats file with header
|
||||
File.write(STATS_FILE, "Timestamp\tCPU(%)\tMemory(MiB)\n")
|
||||
|
||||
spawn do
|
||||
while @running
|
||||
if stat = capture_stats
|
||||
# Append each measurement directly to the file
|
||||
File.open(STATS_FILE, "a") do |file|
|
||||
file.puts "#{stat[:timestamp].to_unix}\t#{stat[:cpu]}\t#{stat[:memory]}"
|
||||
end
|
||||
@stats << stat
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def stop
|
||||
@running = false
|
||||
end
|
||||
|
||||
def avg_stats
|
||||
return {cpu: 0.0, memory: 0.0} if @stats.empty?
|
||||
|
||||
total_cpu = 0.0
|
||||
total_memory = 0.0
|
||||
@stats.each do |stat|
|
||||
total_cpu += stat[:cpu]
|
||||
total_memory += stat[:memory]
|
||||
end
|
||||
|
||||
{
|
||||
cpu: total_cpu / @stats.size,
|
||||
memory: total_memory / @stats.size
|
||||
}
|
||||
end
|
||||
|
||||
private def capture_stats
|
||||
output = IO::Memory.new
|
||||
process = Process.run(
|
||||
"docker", ["stats", "--no-stream", "--format", "{{.CPUPerc}},{{.MemUsage}}", @container_name],
|
||||
output: output
|
||||
)
|
||||
|
||||
if process.success?
|
||||
line = output.to_s.strip
|
||||
parts = line.split(",")
|
||||
if parts.size == 2
|
||||
cpu_part = parts[0].gsub("%", "").to_f
|
||||
|
||||
# Extract the memory value properly by removing the "MiB" suffix
|
||||
mem_string = parts[1].split.first
|
||||
mem_part = mem_string.gsub(/[A-Za-z]+$/, "").to_f
|
||||
|
||||
return {timestamp: Time.utc, cpu: cpu_part, memory: mem_part}
|
||||
end
|
||||
end
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
def check_dependencies
|
||||
{"docker", "jq", "bombardier"}.each do |cmd|
|
||||
process = Process.run("which", [cmd], output: Process::Redirect::Close)
|
||||
unless process.success?
|
||||
puts "Error: #{cmd} is not installed. Please install it to proceed."
|
||||
exit(1)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def setup_containers
|
||||
puts "Setting up..."
|
||||
|
||||
process = Process.run("docker", ["compose", "up", "-d"])
|
||||
unless process.success?
|
||||
puts "Failed to start Docker containers."
|
||||
exit(1)
|
||||
end
|
||||
|
||||
puts "Waiting for the application to be ready..."
|
||||
until begin
|
||||
HTTP::Client.get("#{SERVER_URL}/api/ping").success?
|
||||
rescue
|
||||
false
|
||||
end
|
||||
sleep 1.seconds
|
||||
end
|
||||
|
||||
puts "Seeding the database..."
|
||||
process = Process.run("docker", ["compose", "exec", "app", "sh", "-c", "sqlite3 ./sqlite/data.db < ./db/seed.sql"])
|
||||
|
||||
unless process.success?
|
||||
puts "Error on seeding database"
|
||||
exit(1)
|
||||
end
|
||||
|
||||
puts "Checking seed results..."
|
||||
until begin
|
||||
HTTP::Client.get(
|
||||
"#{API_URL}?limit=1",
|
||||
headers: HTTP::Headers{"X-Api-Key" => API_KEY}
|
||||
).success?
|
||||
rescue
|
||||
false
|
||||
end
|
||||
sleep 2.seconds
|
||||
end
|
||||
end
|
||||
|
||||
def run_benchmark
|
||||
puts "Fetching all created links from /api/links..."
|
||||
|
||||
response = HTTP::Client.get(
|
||||
"#{API_URL}?limit=10000",
|
||||
headers: HTTP::Headers{"X-Api-Key" => API_KEY}
|
||||
)
|
||||
|
||||
sleep 2.seconds
|
||||
unless response.success?
|
||||
puts "Failed to fetch links. Status: #{response.status_code}"
|
||||
exit(1)
|
||||
end
|
||||
|
||||
data = JSON.parse(response.body)
|
||||
links = data["data"].as_a.map { |link| link["refer"].as_s }
|
||||
|
||||
random_link = links.sample
|
||||
puts "Selected link for benchmarking: #{random_link}"
|
||||
|
||||
puts "Starting benchmark with Bombardier..."
|
||||
|
||||
sleep 2.seconds
|
||||
process = Process.new(
|
||||
"bombardier",
|
||||
["-n", NUMBER_OF_REQUESTS.to_s, "-l", "--disableKeepAlives", random_link],
|
||||
output: Process::Redirect::Inherit,
|
||||
error: Process::Redirect::Inherit
|
||||
)
|
||||
|
||||
status = process.wait
|
||||
|
||||
if status.success?
|
||||
puts "Benchmark completed successfully."
|
||||
else
|
||||
puts "Bombardier failed with error code: #{status.exit_code}"
|
||||
exit(1)
|
||||
end
|
||||
end
|
||||
|
||||
def analyze_resource_usage
|
||||
puts "Analyzing resource usage..."
|
||||
|
||||
sleep 2.seconds
|
||||
# Read stats directly from file for more accurate results
|
||||
if File.exists?(STATS_FILE)
|
||||
lines = File.read_lines(STATS_FILE)
|
||||
# Skip header
|
||||
lines = lines[1..-1] if lines.size > 0
|
||||
|
||||
if lines.size > 0
|
||||
total_cpu = 0.0
|
||||
total_memory = 0.0
|
||||
peak_cpu = 0.0
|
||||
peak_memory = 0.0
|
||||
|
||||
lines.each do |line|
|
||||
fields = line.split("\t")
|
||||
if fields.size >= 3
|
||||
begin
|
||||
cpu = fields[1].to_f
|
||||
memory = fields[2].to_f
|
||||
|
||||
total_cpu += cpu
|
||||
total_memory += memory
|
||||
|
||||
# Track peaks in a single pass
|
||||
peak_cpu = cpu if cpu > peak_cpu
|
||||
peak_memory = memory if memory > peak_memory
|
||||
rescue
|
||||
# Skip invalid lines
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
avg_cpu = total_cpu / lines.size
|
||||
avg_memory = total_memory / lines.size
|
||||
|
||||
stats_summary = <<-STATS
|
||||
**** Resource Usage Statistics ****
|
||||
Measurements: #{lines.size}
|
||||
Average CPU Usage: #{avg_cpu.round(2)}%
|
||||
Average Memory Usage: #{avg_memory.round(2)} MiB
|
||||
Peak CPU Usage: #{peak_cpu.round(2)}%
|
||||
Peak Memory Usage: #{peak_memory.round(2)} MiB
|
||||
|
||||
STATS
|
||||
|
||||
File.open(STATS_FILE, "a") do |file|
|
||||
file.puts "\n" + stats_summary
|
||||
end
|
||||
|
||||
puts File.read(STATS_FILE)
|
||||
else
|
||||
puts "No resource usage data collected."
|
||||
end
|
||||
else
|
||||
puts "Resource usage file not found."
|
||||
end
|
||||
end
|
||||
|
||||
def cleanup
|
||||
Process.run("docker", ["compose", "down"])
|
||||
puts "Cleanup completed. Resource usage data saved in #{STATS_FILE}"
|
||||
end
|
||||
|
||||
def main
|
||||
check_dependencies
|
||||
setup_containers
|
||||
|
||||
monitor = ResourceMonitor.new(CONTAINER_NAME)
|
||||
monitor.start
|
||||
|
||||
begin
|
||||
run_benchmark
|
||||
|
||||
monitor.stop
|
||||
analyze_resource_usage
|
||||
ensure
|
||||
cleanup
|
||||
end
|
||||
end
|
||||
|
||||
main
|
||||
Binary file not shown.
+204
-20
@@ -93,6 +93,10 @@ user_agent_parsers:
|
||||
- regex: '(NewRelicPinger)/(\d+)\.(\d+)'
|
||||
family_replacement: 'NewRelicPingerBot'
|
||||
|
||||
# Dynatrace/Ruxit synthetic monitor
|
||||
- regex: '(RuxitSynthetic)/(\d+)\.(\d+)'
|
||||
family_replacement: 'Ruxit Synthetic'
|
||||
|
||||
# Tableau
|
||||
- regex: '(Tableau)/(\d+)\.(\d+)'
|
||||
family_replacement: 'Tableau'
|
||||
@@ -148,7 +152,7 @@ user_agent_parsers:
|
||||
family_replacement: 'Pinterestbot'
|
||||
|
||||
# Bots
|
||||
- regex: '(CSimpleSpider|Cityreview Robot|CrawlDaddy|CrawlFire|Finderbots|Index crawler|Job Roboter|KiwiStatus Spider|Lijit Crawler|QuerySeekerSpider|ScollSpider|Trends Crawler|USyd-NLP-Spider|SiteCat Webbot|BotName\/\$BotVersion|123metaspider-Bot|1470\.net crawler|50\.nu|8bo Crawler Bot|Aboundex|Accoona-[A-z]{1,30}-Agent|AdsBot-Google(?:-[a-z]{1,30}|)|altavista|AppEngine-Google|archive.{0,30}\.org_bot|archiver|Ask Jeeves|[Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]{1,30})(?:-[A-Za-z]{1,30}|)|bingbot|BingPreview|blitzbot|BlogBridge|Bloglovin|BoardReader Blog Indexer|BoardReader Favicon Fetcher|boitho.com-dc|BotSeer|BUbiNG|\b\w{0,30}favicon\w{0,30}\b|\bYeti(?:-[a-z]{1,30}|)|Catchpoint(?: bot|)|[Cc]harlotte|Checklinks|clumboot|Comodo HTTP\(S\) Crawler|Comodo-Webinspector-Crawler|ConveraCrawler|CRAWL-E|CrawlConvera|Daumoa(?:-feedfetcher|)|Feed Seeker Bot|Feedbin|findlinks|Flamingo_SearchEngine|FollowSite Bot|furlbot|Genieo|gigabot|GomezAgent|gonzo1|(?:[a-zA-Z]{1,30}-|)Googlebot(?:-[a-zA-Z]{1,30}|)|GoogleOther|Google SketchUp|grub-client|gsa-crawler|heritrix|HiddenMarket|holmes|HooWWWer|htdig|ia_archiver|ICC-Crawler|Icarus6j|ichiro(?:/mobile|)|IconSurf|IlTrovatore(?:-Setaccio|)|InfuzApp|Innovazion Crawler|InternetArchive|IP2[a-z]{1,30}Bot|jbot\b|KaloogaBot|Kraken|Kurzor|larbin|LEIA|LesnikBot|Linguee Bot|LinkAider|LinkedInBot|Lite Bot|Llaut|lycos|Mail\.RU_Bot|masscan|masidani_bot|Mediapartners-Google|Microsoft .{0,30} Bot|mogimogi|mozDex|MJ12bot|msnbot(?:-media {0,2}|)|msrbot|Mtps Feed Aggregation System|netresearch|Netvibes|NewsGator[^/]{0,30}|^NING|Nutch[^/]{0,30}|Nymesis|ObjectsSearch|OgScrper|Orbiter|OOZBOT|PagePeeker|PagesInventory|PaxleFramework|Peeplo Screenshot Bot|PHPCrawl|PlantyNet_WebRobot|Pompos|Qwantify|Read%20Later|Reaper|RedCarpet|Retreiver|Riddler|Rival IQ|scooter|Scrapy|Scrubby|searchsight|seekbot|semanticdiscovery|SemrushBot|Simpy|SimplePie|SEOstats|SimpleRSS|SiteCon|Slackbot-LinkExpanding|Slack-ImgProxy|Slurp|snappy|Speedy Spider|Squrl Java|Stringer|TheUsefulbot|ThumbShotsBot|Thumbshots\.ru|Tiny Tiny RSS|Twitterbot|WhatsApp|URL2PNG|Vagabondo|VoilaBot|^vortex|Votay bot|^voyager|WASALive.Bot|Web-sniffer|WebThumb|WeSEE:[A-z]{1,30}|WhatWeb|WIRE|WordPress|Wotbox|www\.almaden\.ibm\.com|Xenu(?:.s|) Link Sleuth|Xerka [A-z]{1,30}Bot|yacy(?:bot|)|YahooSeeker|Yahoo! Slurp|Yandex\w{1,30}|YodaoBot(?:-[A-z]{1,30}|)|YottaaMonitor|Yowedo|^Zao|^Zao-Crawler|ZeBot_www\.ze\.bz|ZooShot|ZyBorg|ArcGIS Hub Indexer|GPTBot)(?:[ /]v?(\d+)(?:\.(\d+)(?:\.(\d+)|)|)|)'
|
||||
- regex: '(CSimpleSpider|Cityreview Robot|CrawlDaddy|CrawlFire|Finderbots|Index crawler|Job Roboter|KiwiStatus Spider|Lijit Crawler|QuerySeekerSpider|ScollSpider|Trends Crawler|USyd-NLP-Spider|SiteCat Webbot|BotName\/\$BotVersion|123metaspider-Bot|1470\.net crawler|50\.nu|8bo Crawler Bot|Aboundex|Accoona-[A-z]{1,30}-Agent|AdsBot-Google(?:-[a-z]{1,30}|)|altavista|AppEngine-Google|archive.{0,30}\.org_bot|archiver|Ask Jeeves|[Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]{1,30})(?:-[A-Za-z]{1,30}|)|bingbot|BingPreview|blitzbot|BlogBridge|Bloglovin|BoardReader Blog Indexer|BoardReader Favicon Fetcher|boitho.com-dc|BotSeer|BUbiNG|\b\w{0,30}favicon\w{0,30}\b|\bYeti(?:-[a-z]{1,30}|)|Catchpoint(?: bot|)|[Cc]harlotte|Checklinks|clumboot|Comodo HTTP\(S\) Crawler|Comodo-Webinspector-Crawler|ConveraCrawler|CRAWL-E|CrawlConvera|Daumoa(?:-feedfetcher|)|Feed Seeker Bot|Feedbin|findlinks|Flamingo_SearchEngine|FollowSite Bot|furlbot|Genieo|gigabot|GomezAgent|gonzo1|(?:[a-zA-Z]{1,30}-|)Googlebot(?:-[a-zA-Z]{1,30}|)|GoogleOther|Google SketchUp|grub-client|gsa-crawler|heritrix|HiddenMarket|holmes|HooWWWer|htdig|ia_archiver|ICC-Crawler|Icarus6j|ichiro(?:/mobile|)|IconSurf|IlTrovatore(?:-Setaccio|)|InfuzApp|Innovazion Crawler|InternetArchive|IP2[a-z]{1,30}Bot|jbot\b|KaloogaBot|Kraken|Kurzor|larbin|LEIA|LesnikBot|Linguee Bot|LinkAider|LinkedInBot|Lite Bot|Llaut|lycos|Mail\.RU_Bot|masscan|masidani_bot|Mediapartners-Google|Microsoft .{0,30} Bot|mogimogi|mozDex|MJ12bot|msnbot(?:-media {0,2}|)|msrbot|Mtps Feed Aggregation System|netresearch|Netvibes|NewsGator[^/]{0,30}|^NING|Nutch[^/]{0,30}|Nymesis|ObjectsSearch|OgScrper|Orbiter|OOZBOT|PagePeeker|PagesInventory|PaxleFramework|Peeplo Screenshot Bot|PHPCrawl|PlantyNet_WebRobot|Pompos|Qwantify|Read%20Later|Reaper|RedCarpet|Retreiver|Riddler|Rival IQ|scooter|Scrapy|Scrubby|searchsight|seekbot|semanticdiscovery|SemrushBot|Simpy|SimplePie|SEOstats|SimpleRSS|SiteCon|Slackbot-LinkExpanding|Slack-ImgProxy|Slurp|snappy|Speedy Spider|Squrl Java|Stringer|TheUsefulbot|ThumbShotsBot|Thumbshots\.ru|Tiny Tiny RSS|Twitterbot|WhatsApp|URL2PNG|Vagabondo|VoilaBot|^vortex|Votay bot|^voyager|WASALive.Bot|Web-sniffer|WebThumb|WeSEE:[A-z]{1,30}|WhatWeb|WIRE|WordPress|Wotbox|www\.almaden\.ibm\.com|Xenu(?:.s|) Link Sleuth|Xerka [A-z]{1,30}Bot|yacy(?:bot|)|YahooSeeker|Yahoo! Slurp|Yandex\w{1,30}|YodaoBot(?:-[A-z]{1,30}|)|YottaaMonitor|Yowedo|^Zao|^Zao-Crawler|ZeBot_www\.ze\.bz|ZooShot|ZyBorg|ArcGIS Hub Indexer|GPTBot|Google-InspectionTool)(?:[ /]v?(\d+)(?:\.(\d+)(?:\.(\d+)|)|)|)'
|
||||
|
||||
# AWS S3 Clients
|
||||
# must come before "Bots General matcher" to catch "boto"/"boto3" before "bot"
|
||||
@@ -206,7 +210,12 @@ user_agent_parsers:
|
||||
- regex: '\[(Pinterest)/[^\]]{1,50}\]'
|
||||
- regex: '(Pinterest)(?: for Android(?: Tablet|)|)/(\d+)(?:\.(\d+)|)(?:\.(\d+)|)'
|
||||
# Instagram app
|
||||
# iOS Instagram embeds the token inside a full WebKit UA:
|
||||
# Mozilla/5.0 (iPhone; ...) Mobile/... Instagram VERSION (...)
|
||||
# Android Instagram uses a bare format with no browser wrapper:
|
||||
# Instagram VERSION Android (...)
|
||||
- regex: 'Mozilla.{1,200}Mobile.{1,100}(Instagram).(\d+)\.(\d+)\.(\d+)'
|
||||
- regex: '(Instagram) (\d+)\.(\d+)\.(\d+)'
|
||||
# Flipboard app
|
||||
- regex: 'Mozilla.{1,200}Mobile.{1,100}(Flipboard).(\d+)\.(\d+)\.(\d+)'
|
||||
# Flipboard-briefing app
|
||||
@@ -228,6 +237,9 @@ user_agent_parsers:
|
||||
# KakaoTalk
|
||||
- regex: 'Mozilla.{1,200}Mobile.{1,100}(KAKAOTALK)/(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'KakaoTalk'
|
||||
# Telegram
|
||||
- regex: '(Telegram-Android)/(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'Telegram'
|
||||
|
||||
# Phantom app
|
||||
- regex: 'Mozilla.{1,200}Mobile.{1,100}(Phantom\/ios|Phantom\/android).(\d+)\.(\d+)\.(\d+)'
|
||||
@@ -248,6 +260,10 @@ user_agent_parsers:
|
||||
- regex: '(PaleMoon)/(\d+)\.(\d+)(?:\.(\d+)|)'
|
||||
family_replacement: 'Pale Moon'
|
||||
|
||||
# Camoufox - anti-detect Firefox fork for web scraping/automation; replaces the
|
||||
# Firefox version token with "Camoufox Camoufox VERSION" in the UA string
|
||||
- regex: '(Camoufox) Camoufox (\d+)\.(\d+)'
|
||||
|
||||
# Firefox
|
||||
- regex: '(Fennec)/(\d+)\.(\d+)\.?([ab]?\d+[a-z]*)'
|
||||
family_replacement: 'Firefox Mobile'
|
||||
@@ -296,7 +312,7 @@ user_agent_parsers:
|
||||
|
||||
# UC Browser
|
||||
# we need check it before opera. In other case case UC Browser detected look like Opera Mini
|
||||
- regex: '(UC? ?Browser|UCWEB|U3)[ /]?(\d+)\.(\d+)\.(\d+)'
|
||||
- regex: '(UC? ?Browser|UCWEB|UCMobile|U3)[ /]?(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'UC Browser'
|
||||
|
||||
# Opera will stop at 9.80 and hide the real version in the Version string.
|
||||
@@ -321,6 +337,14 @@ user_agent_parsers:
|
||||
- regex: '(?:Chrome).{1,300}(OPR)/(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'Opera'
|
||||
|
||||
# Opera GX uses "OPX" instead of "OPR"
|
||||
- regex: '(OPX)/(\d+)\.(\d+)(?:\.(\d+)|)'
|
||||
family_replacement: 'Opera GX'
|
||||
|
||||
# Opera Touch uses "OPT"
|
||||
- regex: '(OPT)/(\d+)\.(\d+)(?:\.(\d+)|)'
|
||||
family_replacement: 'Opera Touch'
|
||||
|
||||
# Opera Coast
|
||||
- regex: '(Coast)/(\d+).(\d+).(\d+)'
|
||||
family_replacement: 'Opera Coast'
|
||||
@@ -410,10 +434,14 @@ user_agent_parsers:
|
||||
- regex: '(AlohaBrowser|ABB)/(\d+)\.(\d+)\.(\d+)(?:\.(\d+)|)'
|
||||
family_replacement: 'Aloha Browser'
|
||||
|
||||
# Brave Browser https://brave.com/ , should go before Safari and Chrome Mobile
|
||||
# Brave Browser, should go before Safari and Chrome Mobile
|
||||
- regex: '((?:B|b)rave(?:\sChrome)?)/(\d+)(?:\.(\d+)|)(?:\.(\d+)|)(?:\.(\d+)|)'
|
||||
family_replacement: 'Brave'
|
||||
|
||||
# Brave iOS Browser, checks for (Brave) or Brave at end
|
||||
- regex: '(?:\()?Brave(?:\))?\s*$'
|
||||
family_replacement: 'Brave'
|
||||
|
||||
# Amazon Silk, should go before Safari and Chrome Mobile
|
||||
- regex: '(Silk)/(\d+)\.(\d+)(?:\.([0-9\-]+)|)'
|
||||
family_replacement: 'Amazon Silk'
|
||||
@@ -443,12 +471,6 @@ user_agent_parsers:
|
||||
- regex: '(coc_coc_browser)/(\d+)\.(\d+)(?:\.(\d+)|)'
|
||||
family_replacement: 'Coc Coc'
|
||||
|
||||
# Baidu Browsers (desktop spoofs chrome & IE, explorer is mobile)
|
||||
- regex: '(baidubrowser)[/\s](\d+)(?:\.(\d+)|)(?:\.(\d+)|)'
|
||||
family_replacement: 'Baidu Browser'
|
||||
- regex: '(FlyFlow)/(\d+)\.(\d+)'
|
||||
family_replacement: 'Baidu Explorer'
|
||||
|
||||
# MxBrowser is Maxthon. Must go before Mobile Chrome for Android
|
||||
- regex: '(MxBrowser)/(\d+)\.(\d+)(?:\.(\d+)|)'
|
||||
family_replacement: 'Maxthon'
|
||||
@@ -477,6 +499,12 @@ user_agent_parsers:
|
||||
- regex: 'Mozilla.{1,200}Android.{1,200}(GSA)/(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'Google'
|
||||
|
||||
# Baidu Browsers (desktop spoofs chrome & IE, explorer is mobile)
|
||||
- regex: '(baidubrowser)[/\s](\d+)(?:\.(\d+)|)(?:\.(\d+)|)'
|
||||
family_replacement: 'Baidu Browser'
|
||||
- regex: '(FlyFlow|flyflow|baiduboxapp)/(\d+)\.(\d+)(?:\.(\d+)|)(?:\.(\d+)|)'
|
||||
family_replacement: 'Baidu Explorer'
|
||||
|
||||
# QQ Browsers
|
||||
- regex: '(MQQBrowser/Mini)(?:(\d+)(?:\.(\d+)|)(?:\.(\d+)|)|)'
|
||||
family_replacement: 'QQ Browser Mini'
|
||||
@@ -506,10 +534,34 @@ user_agent_parsers:
|
||||
family_replacement: 'Ecosia Android'
|
||||
|
||||
# VivoBrowser
|
||||
- regex: '(VivoBrowser)\/(\d+)\.(\d+)\.(\d+)\.(\d+)'
|
||||
- regex: '(VivoBrowser)\/(\d+)\.(\d+)\.(\d+)(?:\.(\d+)|)'
|
||||
|
||||
# HiBrowser
|
||||
- regex: '(HiBrowser)\/v(\d+)\.(\d+)\.(\d+)\.(\d+)'
|
||||
- regex: '(H[Ii]Browser)\/v(\d+)\.(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'HiBrowser'
|
||||
|
||||
# Honor Browser
|
||||
- regex: '(HonorBrowser)/(\d+)\.(\d+)\.(\d+)(?:\.(\d+)|)'
|
||||
family_replacement: 'Honor Browser'
|
||||
|
||||
# Honor Browser
|
||||
- regex: '(bdhonorbrowser)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'Honor Browser'
|
||||
|
||||
# HeyTap Browser
|
||||
- regex: '(HeyTapBrowser)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'HeyTap Browser'
|
||||
|
||||
# Weibo
|
||||
# Must before Chrome Mobile WebView
|
||||
- regex: '(weibo)__(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'Weibo'
|
||||
- regex: '(WeiboliteiOS|WeiboIntliOS)'
|
||||
family_replacement: 'Weibo'
|
||||
|
||||
# Phoenix Browser
|
||||
- regex: '(PHX)/(\d+)\.(\d+)'
|
||||
family_replacement: 'Phoenix Browser'
|
||||
|
||||
# Chrome Mobile
|
||||
- regex: 'Version/.{1,300}(Chrome)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
|
||||
@@ -592,7 +644,7 @@ user_agent_parsers:
|
||||
- regex: '(115Browser)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: '115 Browser'
|
||||
|
||||
# Avira
|
||||
# Avira
|
||||
- regex: '(Avira)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'Avira'
|
||||
|
||||
@@ -612,9 +664,9 @@ user_agent_parsers:
|
||||
family_replacement: 'Quark PC'
|
||||
|
||||
# Smart Lenovo Browser
|
||||
- regex: '(SLBrowser)/(\d+)\.(\d+)\.(\d+)\.(\d+) SLBChan/(\d+)'
|
||||
- regex: '(SLBrowser)/(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'Smart Lenovo Browser'
|
||||
|
||||
|
||||
# Atom Browser
|
||||
- regex: '(Atom)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'Atom Browser'
|
||||
@@ -667,6 +719,50 @@ user_agent_parsers:
|
||||
- regex: '(JiSu)/(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'JiSu Browser'
|
||||
|
||||
# Wolvic Browser
|
||||
- regex: '(Wolvic)/(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'Wolvic Browser'
|
||||
|
||||
# SmartTV WebBrowser
|
||||
- regex: '(Thano)/(\d+)\.(\d+)'
|
||||
family_replacement: 'SmartTV WebBrowser'
|
||||
|
||||
# WeChat Browser
|
||||
- regex: '(MicroMessenger)/(\d+)\.(\d+)(?:\.(\d+)|)'
|
||||
family_replacement: 'WeChat Browser'
|
||||
|
||||
# Odin Browser
|
||||
- regex: '(Odin)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'Odin'
|
||||
|
||||
# NetCast Smart TV
|
||||
- regex: '(Colt)/(\d+)\.(\d+)'
|
||||
family_replacement: 'NetCast Smart TV'
|
||||
# Lite Browser
|
||||
- regex: '(Lite Browser)/(\d+)\.(\d+)'
|
||||
family_replacement: 'Lite Browser'
|
||||
|
||||
# Vewd Browser
|
||||
- regex: '(OMI)/(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'Vewd Browser'
|
||||
|
||||
# Mypal
|
||||
- regex: '(Mypal)/(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'Mypal Browser'
|
||||
|
||||
# Chess.com native app
|
||||
- regex: '(Chesscom-Android)/(\d+)\.(\d+)\.(\d+)'
|
||||
|
||||
# Roblox native app
|
||||
- regex: '(RobloxApp)/(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'Roblox App'
|
||||
|
||||
# Roadrunner iOS app (not the legacy Time Warner Cable ISP identifier)
|
||||
- regex: '(Roadrunner)/IOS/\d+/(\d+)\.(\d+)\.(\d+)'
|
||||
|
||||
# Ancestry.com Android app
|
||||
- regex: '(AncestryAndroid)/(\d+)\.(\d+)(?:\.(\d+)|)'
|
||||
|
||||
#### END SPECIAL CASES TOP ####
|
||||
|
||||
#### MAIN CASES - this catches > 50% of all browsers ####
|
||||
@@ -764,6 +860,96 @@ user_agent_parsers:
|
||||
# Browser/major_version.minor_version
|
||||
- regex: '(bingbot|Bolt|AdobeAIR|Jasmine|IceCat|Skyfire|Midori|Maxthon|Lynx|Arora|IBrowse|Dillo|Camino|Shiira|Fennec|Phoenix|Flock|Netscape|Lunascape|Epiphany|WebPilot|Opera Mini|Opera|NetFront|Netfront|Konqueror|Googlebot|SeaMonkey|Kazehakase|Vienna|Iceape|Iceweasel|IceWeasel|Iron|K-Meleon|Sleipnir|Galeon|GranParadiso|iCab|iTunes|MacAppStore|NetNewsWire|Space Bison|Stainless|Orca|Dolfin|BOLT|Minimo|Tizen Browser|Polaris|Abrowser|Planetweb|ICE Browser|mDolphin|qutebrowser|Otter|QupZilla|MailBar|kmail2|YahooMobileMail|ExchangeWebServices|ExchangeServicesClient|Dragon|Outlook-iOS-Android)/(\d+)\.(\d+)(?:\.(\d+)|)'
|
||||
|
||||
# Qt Web Engine embedded browser, must be before Chrome
|
||||
- regex: '(QtWebEngine)/(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'Qt Web Engine'
|
||||
|
||||
# OpenWave browser (Chromium-based), must be before Chrome
|
||||
- regex: '(OpenWave)/(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'Open Wave'
|
||||
|
||||
# AtContent - confirmed APT29/Nobelium (Cozy Bear) C2 malware marker. The implant
|
||||
# (AcroSup.dll, side-loaded via Adobe WCChromeNativeMessagingHost.exe) uses a hardcoded
|
||||
# UA of the form 'Chrome/100.0.4896.75 Safari/537.36 AtContent/91.5.2444.45' to
|
||||
# communicate with Dropbox C2. Also observed appended after Edg/ tokens.
|
||||
# Source: Cluster25/DuskRise 'Cozy Smuggled Into the Box', May 2022
|
||||
# (https://www.duskrise.com/2022/05/13/cozy-smuggled-into-the-box-apt29-abusing-legitimate-software-for-targeted-operations-in-europe/)
|
||||
|
||||
- regex: '(AtContent)/(\d+)\.(\d+)\.(\d+)'
|
||||
# Trailer - suspicious fake UA token appended to Chrome/Edge/Opera UA strings
|
||||
# (TOKEN/MAJOR.MINOR.BUILD.PATCH). No known legitimate browser uses this token.
|
||||
# Structurally identical to AtContent (confirmed APT29/Nobelium C2 marker; see
|
||||
# Cluster25/DuskRise 'Cozy Smuggled Into the Box', May 2022). Unconfirmed attribution;
|
||||
# may be same actor rotating token names or a copycat using the same spoofing technique.
|
||||
- regex: '(Trailer)/(\d+)\.(\d+)\.(\d+)'
|
||||
|
||||
# Agency - suspicious fake UA token appended to Chrome UA strings
|
||||
# (TOKEN/MAJOR.MINOR.BUILD.PATCH). No known legitimate browser uses this token.
|
||||
# Structurally identical to AtContent (confirmed APT29/Nobelium C2 marker; see
|
||||
# Cluster25/DuskRise 'Cozy Smuggled Into the Box', May 2022). Unconfirmed attribution;
|
||||
# may be same actor rotating token names or a copycat using the same spoofing technique.
|
||||
- regex: '(Agency)/(\d+)\.(\d+)\.(\d+)'
|
||||
|
||||
# Herring - suspicious fake UA token appended to Chrome UA strings
|
||||
# (TOKEN/MAJOR.MINOR.BUILD.PATCH). No known legitimate browser uses this token.
|
||||
# Structurally identical to AtContent (confirmed APT29/Nobelium C2 marker; see
|
||||
# Cluster25/DuskRise 'Cozy Smuggled Into the Box', May 2022). Unconfirmed attribution;
|
||||
# may be same actor rotating token names or a copycat using the same spoofing technique.
|
||||
- regex: '(Herring)/(\d+)\.(\d+)\.(\d+)'
|
||||
|
||||
# Config - suspicious fake UA token appended to Chrome UA strings
|
||||
# (TOKEN/MAJOR.MINOR.BUILD.PATCH). No known legitimate browser uses this token.
|
||||
# Structurally identical to AtContent (confirmed APT29/Nobelium C2 marker; see
|
||||
# Cluster25/DuskRise 'Cozy Smuggled Into the Box', May 2022). Unconfirmed attribution;
|
||||
# may be same actor rotating token names or a copycat using the same spoofing technique.
|
||||
- regex: '(Config)/(\d+)\.(\d+)\.(\d+)'
|
||||
|
||||
# Viewer - suspicious fake UA token appended to Chrome UA strings
|
||||
# (TOKEN/MAJOR.MINOR.BUILD.PATCH). No known legitimate browser uses this token.
|
||||
# Structurally identical to AtContent (confirmed APT29/Nobelium C2 marker; see
|
||||
# Cluster25/DuskRise 'Cozy Smuggled Into the Box', May 2022). Unconfirmed attribution;
|
||||
# may be same actor rotating token names or a copycat using the same spoofing technique.
|
||||
- regex: '(Viewer)/(\d+)\.(\d+)\.(\d+)'
|
||||
|
||||
# LikeWise - suspicious fake UA token appended to Chrome UA strings
|
||||
# (TOKEN/MAJOR.MINOR.BUILD.PATCH). No known legitimate browser uses this token.
|
||||
# Structurally identical to AtContent (confirmed APT29/Nobelium C2 marker; see
|
||||
# Cluster25/DuskRise 'Cozy Smuggled Into the Box', May 2022). Unconfirmed attribution;
|
||||
# may be same actor rotating token names or a copycat using the same spoofing technique.
|
||||
- regex: '(LikeWise)/(\d+)\.(\d+)\.(\d+)'
|
||||
|
||||
# Unique - suspicious fake UA token appended to Chrome/Opera UA strings
|
||||
# (TOKEN/MAJOR.MINOR.BUILD.PATCH). No known legitimate browser uses this token.
|
||||
# Structurally identical to AtContent (confirmed APT29/Nobelium C2 marker; see
|
||||
# Cluster25/DuskRise 'Cozy Smuggled Into the Box', May 2022). Unconfirmed attribution;
|
||||
# may be same actor rotating token names or a copycat using the same spoofing technique.
|
||||
- regex: '(Unique)/(\d+)\.(\d+)\.(\d+)'
|
||||
|
||||
# CitizenFX - embedded Chromium browser in FiveM/RedM (GTA V / RDR2 game mod frameworks)
|
||||
- regex: '(CitizenFX)/(\d+)\.(\d+)\.(\d+)'
|
||||
|
||||
# R2Client - R2Games game launcher embedded browser (CEF-based)
|
||||
- regex: '(R2Client)/(\d+)\.(\d+)(?:\.(\d+)|)'
|
||||
|
||||
# OBS Studio embedded browser (CEF-based, used for browser sources/docks)
|
||||
- regex: '(OBS)/(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'OBS Studio'
|
||||
|
||||
# Adobe CEP - embedded Chromium runtime for extension panels in Adobe CC apps
|
||||
- regex: '(AdobeCEP)/(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'Adobe CEP'
|
||||
|
||||
# Steam embedded browsers; version from Chrome. Must be before Chrome.
|
||||
# GameOverlay = in-game overlay browser (Shift+Tab)
|
||||
- regex: 'Valve Steam (GameOverlay).{1,200}Chrome/(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'Steam GameOverlay'
|
||||
# Steam Deck built-in browser
|
||||
- regex: 'Valve Steam (Gamepad)/Steam Deck.{1,200}Chrome/(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'Steam Deck'
|
||||
# Steam desktop client browser
|
||||
- regex: '(Valve(?: Steam|) Client).{1,200}Chrome/(\d+)\.(\d+)\.(\d+)'
|
||||
family_replacement: 'Steam Client'
|
||||
|
||||
# Chrome/Chromium/major_version.minor_version
|
||||
- regex: '(Chromium|Chrome)/(\d+)\.(\d+)(?:\.(\d+)|)(?:\.(\d+)|)'
|
||||
|
||||
@@ -1485,7 +1671,7 @@ os_parsers:
|
||||
# Box Drive and Box Sync on Mac OS X use OSX version numbers, not Darwin
|
||||
- regex: '^Box.{0,200};(Darwin)/(10)\.(1\d)(?:\.(\d+)|)'
|
||||
os_replacement: 'Mac OS X'
|
||||
|
||||
|
||||
##########
|
||||
# Hashicorp API
|
||||
# APN/1.0 HashiCorp/1.0 Terraform/1.8.0 (+https://www.terraform.io) terraform-provider-aws/4.67.0 (+https://registry.terraform.io/providers/hashicorp/aws) aws-sdk-go/1.44.261 (go1.19.8; darwin; arm64)
|
||||
@@ -2013,7 +2199,7 @@ device_parsers:
|
||||
# Mobile Spiders
|
||||
# Catch the mobile crawler before checking for iPhones / Androids.
|
||||
#########
|
||||
- regex: '^.{0,100}?(?:(?:iPhone|Windows CE|Windows Phone|Android).{0,300}(?:(?:Bot|Yeti)-Mobile|YRSpider|BingPreview|bots?/\d|(?:bot|spider)\.html)|AdsBot-Google-Mobile.{0,200}iPhone)'
|
||||
- regex: '^.{0,100}?(?:(?:iPhone|Windows CE|Windows Phone|Android).{0,300}(?:(?:Bot|Yeti)-Mobile|YRSpider|BingPreview|bots?/\d|(?:bot|spider)\.html|Google-InspectionTool)|AdsBot-Google-Mobile.{0,200}iPhone)'
|
||||
regex_flag: 'i'
|
||||
device_replacement: 'Spider'
|
||||
brand_replacement: 'Spider'
|
||||
@@ -3215,7 +3401,7 @@ device_parsers:
|
||||
device_replacement: 'HTC $1'
|
||||
brand_replacement: 'HTC'
|
||||
model_replacement: '$1'
|
||||
- regex: '; {0,2}(ADR6200|ADR6400L|ADR6425LVW|Amaze|DesireS?|EndeavorU|Eris|EVO|Evo\d[A-Z]+|HD2|IncredibleS?|Inspire[A-Z0-9]*|Inspire[A-Z0-9]*|Sensation[A-Z0-9]*|Wildfire)[ _-](.{1,200}?)(?:[/;\)]|Build|MIUI|1\.0)'
|
||||
- regex: '; {0,2}(ADR6200|ADR6400L|ADR6425LVW|Amaze|DesireS?|EndeavorU|Eris|EVO|Evo\d[A-Z]+|HD2|IncredibleS?|Inspire[A-Z0-9]*|Sensation[A-Z0-9]*|Wildfire)[ _-](.{1,200}?)(?:[/;\)]|Build|MIUI|1\.0)'
|
||||
regex_flag: 'i'
|
||||
device_replacement: 'HTC $1 $2'
|
||||
brand_replacement: 'HTC'
|
||||
@@ -5619,7 +5805,6 @@ device_parsers:
|
||||
brand_replacement: 'Asus'
|
||||
model_replacement: '$1'
|
||||
|
||||
|
||||
##########
|
||||
# Bird
|
||||
##########
|
||||
@@ -5819,7 +6004,6 @@ device_parsers:
|
||||
brand_replacement: 'Motorola'
|
||||
model_replacement: '$2'
|
||||
|
||||
|
||||
##########
|
||||
# nintendo
|
||||
##########
|
||||
@@ -6023,7 +6207,7 @@ device_parsers:
|
||||
##########
|
||||
# Spiders (this is a hack...)
|
||||
##########
|
||||
- regex: '^.{0,100}(bot|BUbiNG|zao|borg|DBot|oegp|silk|Xenu|zeal|^NING|CCBot|crawl|htdig|lycos|slurp|teoma|voila|yahoo|Sogou|CiBra|Nutch|^Java/|^JNLP/|Daumoa|Daum|Genieo|ichiro|larbin|pompos|Scrapy|snappy|speedy|spider|msnbot|msrbot|vortex|^vortex|crawler|favicon|indexer|Riddler|scooter|scraper|scrubby|WhatWeb|WinHTTP|bingbot|BingPreview|openbot|gigabot|furlbot|polybot|seekbot|^voyager|archiver|Icarus6j|mogimogi|Netvibes|blitzbot|altavista|charlotte|findlinks|Retreiver|TLSProber|WordPress|SeznamBot|ProoXiBot|wsr\-agent|Squrl Java|EtaoSpider|PaperLiBot|SputnikBot|A6\-Indexer|netresearch|searchsight|baiduspider|YisouSpider|ICC\-Crawler|http%20client|Python-urllib|dataparksearch|converacrawler|Screaming Frog|AppEngine-Google|YahooCacheSystem|fast\-webcrawler|Sogou Pic Spider|semanticdiscovery|Innovazion Crawler|facebookexternalhit|Google.{0,200}/\+/web/snippet|Google-HTTP-Java-Client|BlogBridge|IlTrovatore-Setaccio|InternetArchive|GomezAgent|WebThumbnail|heritrix|NewsGator|PagePeeker|Reaper|ZooShot|holmes|NL-Crawler|Pingdom|StatusCake|WhatsApp|masscan|Google Web Preview|Qwantify|Yeti|OgScrper|RecipeRadar|GPTBot)'
|
||||
- regex: '^.{0,100}(bot|BUbiNG|zao|borg|DBot|oegp|silk|Xenu|zeal|^NING|CCBot|crawl|htdig|lycos|slurp|teoma|voila|yahoo|Sogou|CiBra|Nutch|^Java/|^JNLP/|Daumoa|Daum|Genieo|ichiro|larbin|pompos|Scrapy|snappy|speedy|spider|msnbot|msrbot|vortex|^vortex|crawler|favicon|indexer|Riddler|scooter|scraper|scrubby|WhatWeb|WinHTTP|bingbot|BingPreview|openbot|gigabot|furlbot|polybot|seekbot|^voyager|archiver|Icarus6j|mogimogi|Netvibes|blitzbot|altavista|charlotte|findlinks|Retreiver|TLSProber|WordPress|SeznamBot|ProoXiBot|wsr\-agent|Squrl Java|EtaoSpider|PaperLiBot|SputnikBot|A6\-Indexer|netresearch|searchsight|baiduspider|YisouSpider|ICC\-Crawler|http%20client|Python-urllib|dataparksearch|converacrawler|Screaming Frog|AppEngine-Google|YahooCacheSystem|fast\-webcrawler|Sogou Pic Spider|semanticdiscovery|Innovazion Crawler|facebookexternalhit|Google.{0,200}/\+/web/snippet|Google-HTTP-Java-Client|BlogBridge|IlTrovatore-Setaccio|InternetArchive|GomezAgent|WebThumbnail|heritrix|NewsGator|PagePeeker|Reaper|ZooShot|holmes|NL-Crawler|Pingdom|StatusCake|WhatsApp|masscan|Google Web Preview|Qwantify|Yeti|OgScrper|RecipeRadar|GPTBot|Google-InspectionTool)'
|
||||
regex_flag: 'i'
|
||||
device_replacement: 'Spider'
|
||||
brand_replacement: 'Spider'
|
||||
|
||||
-149
@@ -1,149 +0,0 @@
|
||||
# API Reference
|
||||
|
||||
1. **Ping the API**
|
||||
|
||||
- Endpoint: `GET /api/ping`
|
||||
- Payload: None
|
||||
- Response: 200
|
||||
- Response Example
|
||||
```json
|
||||
{
|
||||
"data": "pong"
|
||||
}
|
||||
```
|
||||
2. **Redirect by Slug**
|
||||
|
||||
- Endpoint: `GET /:slug`
|
||||
- Payload: None
|
||||
- Response: 301
|
||||
|
||||
3. **List All Links**
|
||||
|
||||
- Endpoint: `GET /api/links`
|
||||
- Headers: `X-Api-Key`
|
||||
- Query Parameters:
|
||||
- `limit` (optional): Number of results per page (default: 100)
|
||||
- `cursor` (optional): Pagination cursor from previous response
|
||||
- Response: 200
|
||||
- Response Example
|
||||
```json
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db",
|
||||
"refer": "http://localhost:4000/3wP4BQ",
|
||||
"origin": "https://monocuco.donado.co"
|
||||
}
|
||||
],
|
||||
"pagination": {
|
||||
"has_more": true,
|
||||
"next": "75e0a7f4-9c5e-1235-b546-eb9c5e40f7ac"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
4. **List link by ID**
|
||||
- Endpoint: `GET /api/links/:id`
|
||||
- Headers: `X-Api-Key`
|
||||
- Payload: None
|
||||
- Note: This endpoint returns up to 100 of the most recent clicks. For complete click history, use the `/api/links/:id/clicks` endpoint with pagination.
|
||||
- Response: 200
|
||||
- Response Example
|
||||
```json
|
||||
{
|
||||
"data": {
|
||||
"id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db",
|
||||
"refer": "http://localhost:4000/3wP4BQ",
|
||||
"origin": "https://monocuco.donado.co",
|
||||
"clicks": [
|
||||
{
|
||||
"id": "730e2202-58f9-478c-a24c-f1c561df6716",
|
||||
"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0",
|
||||
"country": "DE",
|
||||
"browser": "Firefox",
|
||||
"os": "Mac OS X",
|
||||
"referer": "Direct",
|
||||
"created_at": "2024-07-12T19:25:22Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
5. **List Clicks for a Link**
|
||||
- Endpoint: `GET /api/links/:id/clicks`
|
||||
- Headers: `X-Api-Key`
|
||||
- Query Parameters:
|
||||
- `limit` (optional): Number of results per page (default: 100)
|
||||
- `cursor` (optional): Pagination cursor from previous response
|
||||
- Response: 200
|
||||
- Response Example
|
||||
```json
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"id": "730e2202-58f9-478c-a24c-f1c561df6716",
|
||||
"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0",
|
||||
"country": "DE",
|
||||
"browser": "Firefox",
|
||||
"os": "Mac OS X",
|
||||
"referer": "Direct",
|
||||
"created_at": "2024-07-12T19:25:22Z"
|
||||
}
|
||||
],
|
||||
"pagination": {
|
||||
"has_more": true,
|
||||
"next": "629e3301-47f8-389b-b24c-f1c561df9825"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
6. **Create new link**
|
||||
- Endpoint: `POST /api/links`
|
||||
- Payload:
|
||||
```json
|
||||
{
|
||||
"url": "https://example.com"
|
||||
}
|
||||
```
|
||||
- Headers: `X-Api-Key`
|
||||
- Response: 201
|
||||
- Response Example:
|
||||
```json
|
||||
{
|
||||
"data": {
|
||||
"id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db",
|
||||
"refer": "http://localhost:4000/3wP4BQ",
|
||||
"origin": "https://example.com",
|
||||
"clicks": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
7. **Update an existing link by ID**
|
||||
- Endpoint: `PUT /api/links/:id`
|
||||
- Payload:
|
||||
```json
|
||||
{
|
||||
"url": "https://newexample.com"
|
||||
}
|
||||
```
|
||||
- Headers: `X-Api-Key`
|
||||
- Response: 200
|
||||
- Response Example:
|
||||
```json
|
||||
{
|
||||
"data": {
|
||||
"id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db",
|
||||
"refer": "http://localhost:4000/3wP4BQ",
|
||||
"origin": "https://newexample.com",
|
||||
"clicks": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
8. **Delete a link by ID**
|
||||
- Endpoint: `DELETE /api/links/:id`
|
||||
- Payload: None
|
||||
- Headers: `X-Api-Key`
|
||||
- Response: 204
|
||||
+127
-108
@@ -6,74 +6,13 @@ Options:
|
||||
--create-user=NAME Create a new user with the given name
|
||||
--list-users List all users
|
||||
--delete-user=USER_ID Delete a user by ID
|
||||
--update-data Download all required data files
|
||||
```
|
||||
|
||||
## Run It Anywhere
|
||||
|
||||
### Docker Compose
|
||||
|
||||
```bash
|
||||
docker-compose up
|
||||
|
||||
# Optional: Generate an api key
|
||||
# docker-compose exec -it app cli --create-user=Admin
|
||||
```
|
||||
|
||||
### Docker CLI
|
||||
|
||||
```bash
|
||||
docker run \
|
||||
--name bit \
|
||||
-p 4000:4000 \
|
||||
-e ENV="production" \
|
||||
-e DATABASE_URL="sqlite3://./sqlite/data.db" \
|
||||
-e APP_URL="http://localhost:4000" \
|
||||
-e ADMIN_NAME="Admin" \
|
||||
-e ADMIN_API_KEY=$(openssl rand -base64 32) \
|
||||
sjdonado/bit
|
||||
|
||||
# Create a new user
|
||||
# docker exec -it bit cli --create-user=Admin
|
||||
```
|
||||
|
||||
### Dokku
|
||||
|
||||
```dockerfile
|
||||
FROM sjdonado/bit
|
||||
```
|
||||
|
||||
```bash
|
||||
dokku apps:create bit
|
||||
|
||||
dokku domains:set bit bit.donado.co
|
||||
dokku letsencrypt:enable bit
|
||||
|
||||
dokku storage:ensure-directory bit-sqlite
|
||||
dokku storage:mount bit /var/lib/dokku/data/storage/bit-sqlite:/usr/src/app/sqlite/
|
||||
|
||||
dokku config:set bit DATABASE_URL="sqlite3://./sqlite/data.db" APP_URL=https://bit.donado.co ADMIN_NAME=Admin ADMIN_API_KEY=$(openssl rand -base64 32)
|
||||
|
||||
dokku ports:add bit http:80:4000
|
||||
dokku ports:add bit https:443:4000
|
||||
|
||||
# Create a new user
|
||||
# dokku run bit cli --create-user=Admin
|
||||
```
|
||||
|
||||
### Dokku (same network)
|
||||
Recommended for lower latency communication (no host network traversal)
|
||||
|
||||
```bash
|
||||
dokku network:create bit-net
|
||||
dokku network:set bit attach-post-create bit-net
|
||||
dokku network:set myapp attach-post-create bit-net
|
||||
--update-parsers Download all required data files
|
||||
```
|
||||
|
||||
## Local Development
|
||||
|
||||
### Requirements
|
||||
- Crystal 1.12+
|
||||
- Crystal 1.18+
|
||||
- Shards package manager
|
||||
- SQLite3
|
||||
|
||||
@@ -109,60 +48,140 @@ ENV=test crystal spec
|
||||
|
||||
## Benchmark
|
||||
|
||||
- Colima: cpu 1, mem 1
|
||||
- SoC: Apple M3 Pro
|
||||
### Run
|
||||
|
||||
```
|
||||
~/p/bit> colima start --cpu 1 --memory 1
|
||||
INFO[0000] starting colima
|
||||
INFO[0000] runtime: docker
|
||||
INFO[0001] starting ... context=vm
|
||||
INFO[0076] provisioning ... context=docker
|
||||
INFO[0077] starting ... context=docker
|
||||
INFO[0077] done
|
||||
~/p/bit> ./benchmark.cr
|
||||
Setting up...
|
||||
Waiting for the application to be ready...
|
||||
Seeding the database...
|
||||
Checking seed results...
|
||||
Fetching all created links from /api/links...
|
||||
Selected link for benchmarking: http://localhost:4000/slug4280
|
||||
Starting benchmark with Bombardier...
|
||||
Bombarding http://localhost:4000/slug4280 with 100000 request(s) using 125 connection(s)
|
||||
100000 / 100000 [==============================================================] 100.00% 6562/s 15s
|
||||
shards build --release --no-debug --progress --stats
|
||||
shards run benchmark
|
||||
```
|
||||
|
||||
### Output
|
||||
|
||||
Chip: Apple M4 Pro. Memory: 24GB
|
||||
|
||||
```
|
||||
1762075350 ~/p/bit> shards build --release --no-debug --progress --stats
|
||||
shards run benchmark
|
||||
Dependencies are satisfied
|
||||
Building: bit
|
||||
Parse: 00:00:00.000652375 ( 1.17MB)
|
||||
Semantic (top level): 00:00:00.419246250 ( 163.45MB)
|
||||
Semantic (new): 00:00:00.001636125 ( 163.45MB)
|
||||
Semantic (type declarations): 00:00:00.019569792 ( 179.45MB)
|
||||
Semantic (abstract def check): 00:00:00.009145125 ( 195.45MB)
|
||||
Semantic (restrictions augmenter): 00:00:00.008421709 ( 195.45MB)
|
||||
Semantic (ivars initializers): 00:00:00.019696584 ( 211.45MB)
|
||||
Semantic (cvars initializers): 00:00:00.106829666 ( 211.50MB)
|
||||
Semantic (main): 00:00:00.649298375 ( 499.88MB)
|
||||
Semantic (cleanup): 00:00:00.000765250 ( 499.88MB)
|
||||
Semantic (recursive struct check): 00:00:00.000752250 ( 499.88MB)
|
||||
Codegen (crystal): 00:00:00.521307417 ( 532.38MB)
|
||||
Codegen (bc+obj): 00:00:00.143842542 ( 532.38MB)
|
||||
Codegen (linking): 00:00:00.236228750 ( 532.38MB)
|
||||
|
||||
Macro runs:
|
||||
- /opt/homebrew/Cellar/crystal/1.18.2/share/crystal/src/ecr/process.cr: reused previous compilation (00:00:00.003593375)
|
||||
|
||||
Codegen (bc+obj):
|
||||
- all previous .o files were reused
|
||||
Building: cli
|
||||
Parse: 00:00:00.000053291 ( 1.17MB)
|
||||
Semantic (top level): 00:00:00.323534167 ( 163.45MB)
|
||||
Semantic (new): 00:00:00.001705500 ( 163.45MB)
|
||||
Semantic (type declarations): 00:00:00.018311958 ( 179.45MB)
|
||||
Semantic (abstract def check): 00:00:00.007766750 ( 195.45MB)
|
||||
Semantic (restrictions augmenter): 00:00:00.005686667 ( 195.45MB)
|
||||
Semantic (ivars initializers): 00:00:00.011239792 ( 211.45MB)
|
||||
Semantic (cvars initializers): 00:00:00.100870833 ( 211.50MB)
|
||||
Semantic (main): 00:00:00.285426750 ( 371.62MB)
|
||||
Semantic (cleanup): 00:00:00.000369875 ( 371.62MB)
|
||||
Semantic (recursive struct check): 00:00:00.000570917 ( 371.62MB)
|
||||
Codegen (crystal): 00:00:00.317534875 ( 387.88MB)
|
||||
Codegen (bc+obj): 00:00:00.097321417 ( 387.88MB)
|
||||
Codegen (linking): 00:00:00.095931000 ( 387.88MB)
|
||||
|
||||
Codegen (bc+obj):
|
||||
- all previous .o files were reused
|
||||
Building: benchmark
|
||||
Parse: 00:00:00.000228500 ( 1.17MB)
|
||||
Semantic (top level): 00:00:00.242174458 ( 147.78MB)
|
||||
Semantic (new): 00:00:00.000863333 ( 147.78MB)
|
||||
Semantic (type declarations): 00:00:00.011527792 ( 147.78MB)
|
||||
Semantic (abstract def check): 00:00:00.031242333 ( 147.78MB)
|
||||
Semantic (restrictions augmenter): 00:00:00.003593583 ( 147.78MB)
|
||||
Semantic (ivars initializers): 00:00:00.006753667 ( 147.78MB)
|
||||
Semantic (cvars initializers): 00:00:00.028373834 ( 195.78MB)
|
||||
Semantic (main): 00:00:00.152039542 ( 243.83MB)
|
||||
Semantic (cleanup): 00:00:00.000249084 ( 243.83MB)
|
||||
Semantic (recursive struct check): 00:00:00.000460417 ( 243.83MB)
|
||||
Codegen (crystal): 00:00:00.075461000 ( 259.83MB)
|
||||
Codegen (bc+obj): 00:00:04.834914333 ( 259.83MB)
|
||||
Codegen (linking): 00:00:00.119920416 ( 259.83MB)
|
||||
|
||||
Codegen (bc+obj):
|
||||
- no previous .o files were reused
|
||||
Dependencies are satisfied
|
||||
Building: benchmark
|
||||
Executing: benchmark
|
||||
Cleaning up benchmark database...
|
||||
Deleted existing database: ./sqlite/data.benchmark.db
|
||||
Database cleanup completed.
|
||||
Running database migrations...
|
||||
Migrating db, current version: 0, target: 20250319192003
|
||||
OK 20240512214223_create_links.sql
|
||||
OK 20240512225208_add_slug_index_to_links.sql
|
||||
OK 20240513115731_create_users.sql
|
||||
OK 20240513130054_add_api_key_index_to_users.sql
|
||||
OK 20240711224103_create_clicks.sql
|
||||
OK 20240714215409_update_slug_size_links.sql
|
||||
OK 20250316102350_add_country_to_clicks.sql
|
||||
OK 20250316111734_replace_unkwown_with_null.sql
|
||||
OK 20250318072657_replace_slug_index_with_covering_index.sql
|
||||
OK 20250319192003_convert_all_tables_text_ids_to_integer.sql
|
||||
Migrations completed successfully.
|
||||
Seeding benchmark database...
|
||||
Database seeded successfully.
|
||||
Starting application: ./bit...
|
||||
Application output will be saved to: app_output.log
|
||||
Application started with PID: 11638
|
||||
Using database: ./sqlite/data.benchmark.db
|
||||
Checking if server is ready at http://localhost:4001...
|
||||
.Server is ready!
|
||||
Fetching links from API...
|
||||
Selected link: http://localhost:4001/slug9391
|
||||
|
||||
Starting benchmark with 100000 requests...
|
||||
Bombarding http://localhost:4001/slug9391 with 100000 request(s) using 125 connection(s)
|
||||
100000 / 100000 [============================================================================] 100.00% 11078/s 9s
|
||||
Done!
|
||||
Statistics Avg Stdev Max
|
||||
Reqs/sec 6609.73 1508.34 13145.76
|
||||
Latency 18.92ms 2.34ms 74.58ms
|
||||
Reqs/sec 11427.28 8889.68 30270.91
|
||||
Latency 11.02ms 6.55ms 53.91ms
|
||||
Latency Distribution
|
||||
50% 18.83ms
|
||||
75% 20.19ms
|
||||
90% 21.80ms
|
||||
95% 23.10ms
|
||||
99% 26.54ms
|
||||
50% 1.85ms
|
||||
75% 5.37ms
|
||||
90% 39.36ms
|
||||
95% 39.87ms
|
||||
99% 42.66ms
|
||||
HTTP codes:
|
||||
1xx - 0, 2xx - 0, 3xx - 100000, 4xx - 0, 5xx - 0
|
||||
others - 0
|
||||
Throughput: 1.80MB/s
|
||||
Throughput: 3.08MB/s
|
||||
|
||||
Benchmark completed successfully.
|
||||
Analyzing resource usage...
|
||||
Timestamp CPU(%) Memory(MiB)
|
||||
1742732843 0.02 44.71
|
||||
1742732845 0.02 44.71
|
||||
1742732847 85.34 69.55
|
||||
1742732849 83.5 69.93
|
||||
1742732851 84.26 69.97
|
||||
1742732853 83.64 70.01
|
||||
1742732855 84.23 70.04
|
||||
1742732857 86.41 69.17
|
||||
1742732859 85.77 69.2
|
||||
1742732861 59.67 68.55
|
||||
|
||||
**** Resource Usage Statistics ****
|
||||
Measurements: 10
|
||||
Average CPU Usage: 65.29%
|
||||
Average Memory Usage: 64.58 MiB
|
||||
Peak CPU Usage: 86.41%
|
||||
Peak Memory Usage: 70.04 MiB
|
||||
Cleanup completed. Resource usage data saved in resource_usage.txt
|
||||
Measurements: 12
|
||||
Average CPU Usage: 71.5%
|
||||
Average Memory Usage: 39.8 MiB
|
||||
Peak CPU Usage: 100.0%
|
||||
Peak Memory Usage: 53.41 MiB
|
||||
|
||||
**** Files Generated ****
|
||||
Resource stats: resource_usage.log
|
||||
Application log: app_output.log
|
||||
Database: ./sqlite/data.benchmark.db
|
||||
|
||||
Stopping application...
|
||||
Application stopped.
|
||||
```
|
||||
|
||||
@@ -0,0 +1,503 @@
|
||||
openapi: 3.0.3
|
||||
info:
|
||||
title: Bit - URL Shortener API
|
||||
description: |
|
||||
Fast, lightweight, self-hosted URL shortener service with minimal click tracking.
|
||||
|
||||
## Getting Started
|
||||
|
||||
For setup instructions, please check the [README](https://github.com/sjdonado/bit/blob/master/README.md).
|
||||
|
||||
## Authentication
|
||||
|
||||
Multiple users are supported via `X-Api-Key` headers. Create, list and delete keys via the [CLI](https://github.com/sjdonado/bit/blob/master/SETUP.md#cli).
|
||||
version: 1.6.0
|
||||
contact:
|
||||
name: sjdonado
|
||||
url: https://sjdonado.com
|
||||
|
||||
servers:
|
||||
- url: http://localhost:4000
|
||||
description: Development server
|
||||
|
||||
security:
|
||||
- ApiKeyAuth: []
|
||||
|
||||
paths:
|
||||
/api/ping:
|
||||
get:
|
||||
summary: Ping the API
|
||||
description: Health check endpoint to verify the API is running
|
||||
operationId: ping
|
||||
tags:
|
||||
- Health
|
||||
security: []
|
||||
responses:
|
||||
'200':
|
||||
description: API is healthy
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
data:
|
||||
type: string
|
||||
example: pong
|
||||
|
||||
/{slug}:
|
||||
get:
|
||||
summary: Redirect by slug
|
||||
description: Redirects to the original URL and tracks the click asynchronously
|
||||
operationId: redirectBySlug
|
||||
tags:
|
||||
- Redirects
|
||||
security: []
|
||||
parameters:
|
||||
- name: slug
|
||||
in: path
|
||||
required: true
|
||||
description: The short URL slug
|
||||
schema:
|
||||
type: string
|
||||
example: 3wP4BQ
|
||||
- name: utm_source
|
||||
in: query
|
||||
required: false
|
||||
description: UTM source parameter for tracking
|
||||
schema:
|
||||
type: string
|
||||
example: email_campaign
|
||||
responses:
|
||||
'301':
|
||||
description: Redirect to original URL
|
||||
headers:
|
||||
Location:
|
||||
description: The original URL
|
||||
schema:
|
||||
type: string
|
||||
example: https://example.com
|
||||
X-Forwarded-For:
|
||||
description: Client IP address
|
||||
schema:
|
||||
type: string
|
||||
User-Agent:
|
||||
description: User agent string
|
||||
schema:
|
||||
type: string
|
||||
'404':
|
||||
description: Link not found
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Error'
|
||||
|
||||
/api/links:
|
||||
get:
|
||||
summary: List all links
|
||||
description: Retrieve all links for the authenticated user with pagination support
|
||||
operationId: listLinks
|
||||
tags:
|
||||
- Links
|
||||
parameters:
|
||||
- name: limit
|
||||
in: query
|
||||
description: Number of results per page
|
||||
schema:
|
||||
type: integer
|
||||
default: 100
|
||||
minimum: 1
|
||||
maximum: 1000
|
||||
- name: cursor
|
||||
in: query
|
||||
description: Pagination cursor from previous response
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
'200':
|
||||
description: List of links
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
data:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/components/schemas/LinkSummary'
|
||||
pagination:
|
||||
$ref: '#/components/schemas/Pagination'
|
||||
'401':
|
||||
description: Unauthorized
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Error'
|
||||
|
||||
post:
|
||||
summary: Create new link
|
||||
description: Create a new shortened link
|
||||
operationId: createLink
|
||||
tags:
|
||||
- Links
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
required:
|
||||
- url
|
||||
properties:
|
||||
url:
|
||||
type: string
|
||||
format: uri
|
||||
description: The URL to shorten
|
||||
example: https://example.com
|
||||
responses:
|
||||
'201':
|
||||
description: Link created successfully
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
data:
|
||||
$ref: '#/components/schemas/Link'
|
||||
'400':
|
||||
description: Bad request - invalid URL or missing field
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Error'
|
||||
examples:
|
||||
missingField:
|
||||
value:
|
||||
error: "url: Required field"
|
||||
invalidUrl:
|
||||
value:
|
||||
errors:
|
||||
url:
|
||||
- is invalid
|
||||
'401':
|
||||
description: Unauthorized
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Error'
|
||||
|
||||
/api/links/{id}:
|
||||
get:
|
||||
summary: Get link by ID
|
||||
description: Retrieve a specific link with up to 100 most recent clicks. For complete click history, use /api/links/{id}/clicks
|
||||
operationId: getLink
|
||||
tags:
|
||||
- Links
|
||||
parameters:
|
||||
- name: id
|
||||
in: path
|
||||
required: true
|
||||
description: Link ID
|
||||
schema:
|
||||
type: integer
|
||||
format: int64
|
||||
responses:
|
||||
'200':
|
||||
description: Link details
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
data:
|
||||
$ref: '#/components/schemas/Link'
|
||||
'404':
|
||||
description: Link not found
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Error'
|
||||
'401':
|
||||
description: Unauthorized
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Error'
|
||||
|
||||
put:
|
||||
summary: Update link
|
||||
description: Update the URL of an existing link
|
||||
operationId: updateLink
|
||||
tags:
|
||||
- Links
|
||||
parameters:
|
||||
- name: id
|
||||
in: path
|
||||
required: true
|
||||
description: Link ID
|
||||
schema:
|
||||
type: integer
|
||||
format: int64
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
required:
|
||||
- url
|
||||
properties:
|
||||
url:
|
||||
type: string
|
||||
format: uri
|
||||
description: The new URL
|
||||
example: https://newexample.com
|
||||
responses:
|
||||
'200':
|
||||
description: Link updated successfully
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
data:
|
||||
$ref: '#/components/schemas/Link'
|
||||
'400':
|
||||
description: Bad request
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Error'
|
||||
'401':
|
||||
description: Unauthorized
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Error'
|
||||
'403':
|
||||
description: Forbidden - link belongs to another user
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Error'
|
||||
'404':
|
||||
description: Link not found
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Error'
|
||||
|
||||
delete:
|
||||
summary: Delete link
|
||||
description: Delete a link and all its associated clicks
|
||||
operationId: deleteLink
|
||||
tags:
|
||||
- Links
|
||||
parameters:
|
||||
- name: id
|
||||
in: path
|
||||
required: true
|
||||
description: Link ID
|
||||
schema:
|
||||
type: integer
|
||||
format: int64
|
||||
responses:
|
||||
'204':
|
||||
description: Link deleted successfully
|
||||
'401':
|
||||
description: Unauthorized
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Error'
|
||||
'403':
|
||||
description: Forbidden - link belongs to another user
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Error'
|
||||
'404':
|
||||
description: Link not found
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Error'
|
||||
|
||||
/api/links/{id}/clicks:
|
||||
get:
|
||||
summary: List clicks for a link
|
||||
description: Retrieve all clicks for a specific link with pagination support
|
||||
operationId: listClicks
|
||||
tags:
|
||||
- Clicks
|
||||
parameters:
|
||||
- name: id
|
||||
in: path
|
||||
required: true
|
||||
description: Link ID
|
||||
schema:
|
||||
type: integer
|
||||
format: int64
|
||||
- name: limit
|
||||
in: query
|
||||
description: Number of results per page
|
||||
schema:
|
||||
type: integer
|
||||
default: 100
|
||||
minimum: 1
|
||||
maximum: 1000
|
||||
- name: cursor
|
||||
in: query
|
||||
description: Pagination cursor from previous response
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
'200':
|
||||
description: List of clicks
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
data:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/components/schemas/Click'
|
||||
pagination:
|
||||
$ref: '#/components/schemas/Pagination'
|
||||
'401':
|
||||
description: Unauthorized
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Error'
|
||||
'404':
|
||||
description: Link not found
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Error'
|
||||
|
||||
components:
|
||||
securitySchemes:
|
||||
ApiKeyAuth:
|
||||
type: apiKey
|
||||
in: header
|
||||
name: X-Api-Key
|
||||
description: API key for authentication
|
||||
|
||||
schemas:
|
||||
LinkSummary:
|
||||
type: object
|
||||
properties:
|
||||
id:
|
||||
type: integer
|
||||
format: int64
|
||||
description: Unique link identifier
|
||||
example: 1
|
||||
refer:
|
||||
type: string
|
||||
format: uri
|
||||
description: The shortened URL
|
||||
example: http://localhost:4000/3wP4BQ
|
||||
origin:
|
||||
type: string
|
||||
format: uri
|
||||
description: The original URL
|
||||
example: https://monocuco.donado.co
|
||||
|
||||
Link:
|
||||
allOf:
|
||||
- $ref: '#/components/schemas/LinkSummary'
|
||||
- type: object
|
||||
properties:
|
||||
clicks:
|
||||
type: array
|
||||
description: Array of click records (up to 100 most recent)
|
||||
items:
|
||||
$ref: '#/components/schemas/Click'
|
||||
|
||||
Click:
|
||||
type: object
|
||||
properties:
|
||||
id:
|
||||
type: integer
|
||||
format: int64
|
||||
description: Unique click identifier
|
||||
example: 1
|
||||
user_agent:
|
||||
type: string
|
||||
description: User agent string
|
||||
example: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0
|
||||
country:
|
||||
type: string
|
||||
description: Country code (ISO 3166-1 alpha-2)
|
||||
example: US
|
||||
nullable: true
|
||||
browser:
|
||||
type: string
|
||||
description: Browser name
|
||||
example: Firefox
|
||||
nullable: true
|
||||
os:
|
||||
type: string
|
||||
description: Operating system
|
||||
example: Mac OS X
|
||||
nullable: true
|
||||
referer:
|
||||
type: string
|
||||
description: Referer domain or utm_source
|
||||
example: Direct
|
||||
nullable: true
|
||||
created_at:
|
||||
type: string
|
||||
format: date-time
|
||||
description: Click timestamp
|
||||
example: 2024-07-12T19:25:22Z
|
||||
|
||||
Pagination:
|
||||
type: object
|
||||
properties:
|
||||
has_more:
|
||||
type: boolean
|
||||
description: Whether there are more results
|
||||
example: true
|
||||
next:
|
||||
type: integer
|
||||
format: int64
|
||||
description: Cursor for next page (link/click ID)
|
||||
example: 12
|
||||
nullable: true
|
||||
|
||||
Error:
|
||||
type: object
|
||||
properties:
|
||||
error:
|
||||
type: string
|
||||
description: Error message
|
||||
example: Resource not found
|
||||
required:
|
||||
- error
|
||||
|
||||
ValidationErrors:
|
||||
type: object
|
||||
properties:
|
||||
errors:
|
||||
type: object
|
||||
additionalProperties:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
description: Field-level validation errors
|
||||
example:
|
||||
url:
|
||||
- is invalid
|
||||
|
||||
tags:
|
||||
- name: Health
|
||||
description: Health check endpoints
|
||||
- name: Redirects
|
||||
description: URL redirection and click tracking
|
||||
- name: Links
|
||||
description: Link management operations
|
||||
- name: Clicks
|
||||
description: Click analytics and tracking
|
||||
@@ -0,0 +1,341 @@
|
||||
#!/usr/bin/env crystal
|
||||
|
||||
require "http/client"
|
||||
require "json"
|
||||
|
||||
PORT = "4001"
|
||||
APP_URL = "http://localhost:#{PORT}"
|
||||
API_URL = "#{APP_URL}/api/links"
|
||||
API_KEY = "secure_api_key_1"
|
||||
NUMBER_OF_REQUESTS = 100000
|
||||
|
||||
APP_COMMAND = "./bit"
|
||||
APP_ARGS = [] of String
|
||||
STATS_FILE = "resource_usage.log"
|
||||
APP_LOG_FILE = "app_output.log"
|
||||
|
||||
DATABASE_URL = "sqlite3://./sqlite/data.benchmark.db?journal_mode=wal&synchronous=normal&foreign_keys=true"
|
||||
DATABASE_FILE = "./sqlite/data.benchmark.db"
|
||||
|
||||
class ResourceMonitor
|
||||
def initialize(@pid : Int32)
|
||||
@running = false
|
||||
@stats = [] of {timestamp: Time, cpu: Float64, memory: Float64}
|
||||
end
|
||||
|
||||
def start
|
||||
@running = true
|
||||
@stats.clear
|
||||
File.write(STATS_FILE, "Timestamp\tCPU(%)\tMemory(MiB)\n")
|
||||
|
||||
spawn do
|
||||
while @running
|
||||
if stat = capture_stats
|
||||
File.open(STATS_FILE, "a") do |file|
|
||||
file.puts "#{stat[:timestamp].to_unix}\t#{stat[:cpu]}\t#{stat[:memory]}"
|
||||
end
|
||||
@stats << stat
|
||||
end
|
||||
sleep 1.seconds
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def stop
|
||||
@running = false
|
||||
sleep 1.seconds
|
||||
end
|
||||
|
||||
private def capture_stats
|
||||
output = IO::Memory.new
|
||||
process = Process.run(
|
||||
"ps", ["-p", @pid.to_s, "-o", "%cpu,%mem,rss"],
|
||||
output: output
|
||||
)
|
||||
|
||||
if process.success?
|
||||
lines = output.to_s.strip.split("\n")
|
||||
if lines.size >= 2
|
||||
data_line = lines[1].strip.split
|
||||
if data_line.size >= 3
|
||||
cpu = data_line[0].to_f
|
||||
# RSS is in KB on macOS, convert to MiB
|
||||
memory_kb = data_line[2].to_f
|
||||
memory_mib = memory_kb / 1024.0
|
||||
|
||||
return {timestamp: Time.utc, cpu: cpu, memory: memory_mib}
|
||||
end
|
||||
end
|
||||
end
|
||||
nil
|
||||
end
|
||||
|
||||
def print_summary
|
||||
return if @stats.empty?
|
||||
|
||||
total_cpu = 0.0
|
||||
total_memory = 0.0
|
||||
peak_cpu = 0.0
|
||||
peak_memory = 0.0
|
||||
|
||||
@stats.each do |stat|
|
||||
total_cpu += stat[:cpu]
|
||||
total_memory += stat[:memory]
|
||||
peak_cpu = stat[:cpu] if stat[:cpu] > peak_cpu
|
||||
peak_memory = stat[:memory] if stat[:memory] > peak_memory
|
||||
end
|
||||
|
||||
avg_cpu = total_cpu / @stats.size
|
||||
avg_memory = total_memory / @stats.size
|
||||
|
||||
summary = <<-STATS
|
||||
|
||||
**** Resource Usage Statistics ****
|
||||
Measurements: #{@stats.size}
|
||||
Average CPU Usage: #{avg_cpu.round(2)}%
|
||||
Average Memory Usage: #{avg_memory.round(2)} MiB
|
||||
Peak CPU Usage: #{peak_cpu.round(2)}%
|
||||
Peak Memory Usage: #{peak_memory.round(2)} MiB
|
||||
|
||||
STATS
|
||||
|
||||
File.open(STATS_FILE, "a") do |file|
|
||||
file.puts summary
|
||||
end
|
||||
|
||||
puts summary
|
||||
end
|
||||
end
|
||||
|
||||
def start_application : Process
|
||||
puts "Starting application: #{APP_COMMAND}..."
|
||||
puts "Application output will be saved to: #{APP_LOG_FILE}"
|
||||
|
||||
log_file = File.open(APP_LOG_FILE, "w")
|
||||
|
||||
process = Process.new(
|
||||
APP_COMMAND,
|
||||
APP_ARGS,
|
||||
output: log_file,
|
||||
error: log_file,
|
||||
env: {
|
||||
"DATABASE_URL" => DATABASE_URL,
|
||||
"APP_URL" => APP_URL,
|
||||
"PORT" => PORT,
|
||||
}
|
||||
)
|
||||
|
||||
puts "Application started with PID: #{process.pid}"
|
||||
puts "Using database: #{DATABASE_FILE}"
|
||||
process
|
||||
end
|
||||
|
||||
def stop_application(process : Process)
|
||||
puts "\nStopping application..."
|
||||
process.signal(Signal::TERM)
|
||||
|
||||
# Give it a few seconds to shut down gracefully
|
||||
sleep 3.seconds
|
||||
|
||||
# Force kill if still running
|
||||
begin
|
||||
process.signal(Signal::KILL)
|
||||
rescue
|
||||
# Process already terminated
|
||||
end
|
||||
|
||||
puts "Application stopped."
|
||||
end
|
||||
|
||||
def check_dependencies
|
||||
{"bombardier", "sqlite3", "micrate"}.each do |cmd|
|
||||
process = Process.run("which", [cmd], output: Process::Redirect::Close)
|
||||
unless process.success?
|
||||
puts "Error: #{cmd} is not installed. Please install it to proceed."
|
||||
case cmd
|
||||
when "bombardier"
|
||||
puts " brew install bombardier"
|
||||
when "sqlite3"
|
||||
puts " brew install sqlite3"
|
||||
when "micrate"
|
||||
puts " shards install"
|
||||
end
|
||||
exit(1)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def wait_for_server
|
||||
puts "Checking if server is ready at #{APP_URL}..."
|
||||
|
||||
30.times do
|
||||
begin
|
||||
if HTTP::Client.get("#{APP_URL}/api/ping").success?
|
||||
puts "Server is ready!"
|
||||
return
|
||||
end
|
||||
rescue
|
||||
# Server not ready yet
|
||||
end
|
||||
sleep 1.seconds
|
||||
print "."
|
||||
end
|
||||
|
||||
puts "\nError: Server is not responding. Please start your application first."
|
||||
exit(1)
|
||||
end
|
||||
|
||||
def run_benchmark
|
||||
puts "Fetching links from API..."
|
||||
|
||||
response = HTTP::Client.get(
|
||||
"#{API_URL}?limit=10000",
|
||||
headers: HTTP::Headers{"X-Api-Key" => API_KEY}
|
||||
)
|
||||
|
||||
unless response.success?
|
||||
puts "Failed to fetch links. Status: #{response.status_code}"
|
||||
puts "Make sure your server is running and the API key is correct."
|
||||
exit(1)
|
||||
end
|
||||
|
||||
data = JSON.parse(response.body)
|
||||
links = data["data"].as_a.map { |link| link["refer"].as_s }
|
||||
|
||||
if links.empty?
|
||||
puts "No links found. Please seed your database first."
|
||||
exit(1)
|
||||
end
|
||||
|
||||
random_link = links.sample
|
||||
puts "Selected link: #{random_link}"
|
||||
puts "\nStarting benchmark with #{NUMBER_OF_REQUESTS} requests..."
|
||||
|
||||
sleep 2.seconds
|
||||
|
||||
process = Process.new(
|
||||
"bombardier",
|
||||
["-n", NUMBER_OF_REQUESTS.to_s, "-l", "--disableKeepAlives", random_link],
|
||||
output: Process::Redirect::Inherit,
|
||||
error: Process::Redirect::Inherit
|
||||
)
|
||||
|
||||
status = process.wait
|
||||
|
||||
if status.success?
|
||||
puts "\nBenchmark completed successfully."
|
||||
else
|
||||
puts "\nBombardier failed with error code: #{status.exit_code}"
|
||||
exit(1)
|
||||
end
|
||||
end
|
||||
|
||||
def cleanup_database
|
||||
puts "Cleaning up benchmark database..."
|
||||
|
||||
if File.exists?(DATABASE_FILE)
|
||||
File.delete(DATABASE_FILE)
|
||||
puts "Deleted existing database: #{DATABASE_FILE}"
|
||||
end
|
||||
|
||||
# Also remove WAL and SHM files if they exist
|
||||
["#{DATABASE_FILE}-wal", "#{DATABASE_FILE}-shm"].each do |file|
|
||||
if File.exists?(file)
|
||||
File.delete(file)
|
||||
puts "Deleted: #{file}"
|
||||
end
|
||||
end
|
||||
|
||||
# Ensure sqlite directory exists
|
||||
Dir.mkdir_p("./sqlite")
|
||||
puts "Database cleanup completed."
|
||||
end
|
||||
|
||||
def run_migrations
|
||||
puts "Running database migrations..."
|
||||
|
||||
process = Process.run("which", ["micrate"], output: Process::Redirect::Close)
|
||||
unless process.success?
|
||||
puts "Error: micrate is not installed. Please install it to proceed."
|
||||
puts " shards install"
|
||||
exit(1)
|
||||
end
|
||||
|
||||
process = Process.run(
|
||||
"micrate",
|
||||
["up"],
|
||||
env: {"DATABASE_URL" => DATABASE_URL},
|
||||
output: Process::Redirect::Inherit,
|
||||
error: Process::Redirect::Inherit
|
||||
)
|
||||
|
||||
if process.success?
|
||||
puts "Migrations completed successfully."
|
||||
else
|
||||
puts "Error: Migrations failed."
|
||||
exit(1)
|
||||
end
|
||||
end
|
||||
|
||||
def seed_database
|
||||
puts "Seeding benchmark database..."
|
||||
|
||||
unless File.exists?("./db/seed.sql")
|
||||
puts "Warning: ./db/seed.sql not found. Skipping database seeding."
|
||||
return
|
||||
end
|
||||
|
||||
unless File.exists?(DATABASE_FILE)
|
||||
puts "Warning: #{DATABASE_FILE} not found. Database may not be initialized."
|
||||
end
|
||||
|
||||
process = Process.run(
|
||||
"sqlite3",
|
||||
[DATABASE_FILE],
|
||||
input: File.open("./db/seed.sql"),
|
||||
output: Process::Redirect::Inherit,
|
||||
error: Process::Redirect::Inherit
|
||||
)
|
||||
|
||||
if process.success?
|
||||
puts "Database seeded successfully."
|
||||
else
|
||||
puts "Warning: Database seeding failed. Continuing anyway..."
|
||||
end
|
||||
end
|
||||
|
||||
def main
|
||||
check_dependencies
|
||||
|
||||
# Setup benchmark database
|
||||
cleanup_database
|
||||
run_migrations
|
||||
seed_database
|
||||
|
||||
app_process = start_application
|
||||
|
||||
begin
|
||||
wait_for_server
|
||||
|
||||
# Give it a moment to settle
|
||||
sleep 2.seconds
|
||||
|
||||
monitor = ResourceMonitor.new(app_process.pid.to_i32)
|
||||
monitor.start
|
||||
|
||||
run_benchmark
|
||||
|
||||
monitor.stop
|
||||
monitor.print_summary
|
||||
|
||||
puts "\n**** Files Generated ****"
|
||||
puts " Resource stats: #{STATS_FILE}"
|
||||
puts " Application log: #{APP_LOG_FILE}"
|
||||
puts " Database: #{DATABASE_FILE}"
|
||||
ensure
|
||||
# Always stop the application
|
||||
stop_application(app_process)
|
||||
end
|
||||
end
|
||||
|
||||
main
|
||||
+4
-4
@@ -18,11 +18,11 @@ OptionParser.parse do |parser|
|
||||
exit
|
||||
end
|
||||
|
||||
parser.on("--update-data", "Download all required data files (UA Parser and GeoLite2)") do
|
||||
parser.on("--update-parsers", "Download UA regexes and/or GeoLite2 database") do
|
||||
puts "=== Starting data files update ==="
|
||||
App::Services::Cli.update_uap_regexes
|
||||
App::Services::Cli.download_geolite_db
|
||||
puts "=== All data files updated successfully ==="
|
||||
App::Services::Cli.update_geolite_db
|
||||
puts "=== Data files updated successfully ==="
|
||||
exit
|
||||
end
|
||||
|
||||
@@ -32,6 +32,6 @@ OptionParser.parse do |parser|
|
||||
puts " --create-user=NAME Create a new user with the given name"
|
||||
puts " --list-users List all users"
|
||||
puts " --delete-user=USER_ID Delete a user by ID"
|
||||
puts " --update-data Download all required data files"
|
||||
puts " --update-parsers Download all required data files"
|
||||
end
|
||||
end
|
||||
|
||||
+4
-4
@@ -2,11 +2,11 @@ version: 2.0
|
||||
shards:
|
||||
backtracer:
|
||||
git: https://github.com/sija/backtracer.cr.git
|
||||
version: 1.2.2
|
||||
version: 1.2.4
|
||||
|
||||
crecto:
|
||||
git: https://github.com/fridgerator/crecto.git
|
||||
version: 0.12.1
|
||||
version: 0.14.0
|
||||
|
||||
db:
|
||||
git: https://github.com/crystal-lang/crystal-db.git
|
||||
@@ -18,7 +18,7 @@ shards:
|
||||
|
||||
exception_page:
|
||||
git: https://github.com/crystal-loot/exception_page.git
|
||||
version: 0.4.1
|
||||
version: 0.5.0
|
||||
|
||||
ipaddress:
|
||||
git: https://github.com/sija/ipaddress.cr.git
|
||||
@@ -26,7 +26,7 @@ shards:
|
||||
|
||||
kemal:
|
||||
git: https://github.com/kemalcr/kemal.git
|
||||
version: 1.5.0
|
||||
version: 1.7.3
|
||||
|
||||
maxminddb:
|
||||
git: https://github.com/delef/maxminddb.cr.git
|
||||
|
||||
@@ -1,18 +1,21 @@
|
||||
name: bit
|
||||
version: 1.5.2
|
||||
version: 1.6.0
|
||||
|
||||
authors:
|
||||
- Juan Rodriguez <sjdonado@icloud.com>
|
||||
- Juan Rodriguez Donado <@sjdonado>
|
||||
|
||||
targets:
|
||||
bit:
|
||||
main: bit.cr
|
||||
cli:
|
||||
main: scripts/cli.cr
|
||||
benchmark:
|
||||
main: scripts/benchmark.cr
|
||||
|
||||
dependencies:
|
||||
kemal:
|
||||
github: kemalcr/kemal
|
||||
version: 1.7.3
|
||||
sqlite3:
|
||||
github: crystal-lang/crystal-sqlite3
|
||||
crecto:
|
||||
@@ -29,6 +32,6 @@ development_dependencies:
|
||||
spec-kemal:
|
||||
github: kemalcr/spec-kemal
|
||||
|
||||
crystal: ">= 1.12.1"
|
||||
crystal: ">= 1.18.2"
|
||||
|
||||
license: MIT
|
||||
|
||||
@@ -15,7 +15,7 @@ describe "App::Controllers::Link" do
|
||||
body: payload.to_json
|
||||
)
|
||||
|
||||
parsed_response = Hash(String, Hash(String, String | Int64 | Array(Hash(String, String)))).from_json(response.body)
|
||||
parsed_response = Hash(String, Hash(String, String | Int64 | Array(Hash(String, String | Int64)))).from_json(response.body)
|
||||
parsed_response["data"]["origin"].should eq(payload["url"])
|
||||
end
|
||||
|
||||
@@ -87,11 +87,10 @@ describe "App::Controllers::Link" do
|
||||
test_user = create_test_user()
|
||||
|
||||
test_link = create_test_link(test_user, link)
|
||||
serialized_link = App::Serializers::Link.new(test_link)
|
||||
|
||||
user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0"
|
||||
|
||||
get(serialized_link.refer, headers: HTTP::Headers{
|
||||
get("/#{test_link.slug}", headers: HTTP::Headers{
|
||||
"X-Api-Key" => test_user.api_key.to_s,
|
||||
"User-Agent" => user_agent
|
||||
})
|
||||
@@ -106,12 +105,11 @@ describe "App::Controllers::Link" do
|
||||
test_user = create_test_user()
|
||||
|
||||
test_link = create_test_link(test_user, link)
|
||||
serialized_link = App::Serializers::Link.new(test_link)
|
||||
|
||||
user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0"
|
||||
referer = "https://example.com/page"
|
||||
|
||||
get(serialized_link.refer, headers: HTTP::Headers{
|
||||
get("/#{test_link.slug}", headers: HTTP::Headers{
|
||||
"User-Agent" => user_agent,
|
||||
"Referer" => referer
|
||||
})
|
||||
@@ -121,7 +119,7 @@ describe "App::Controllers::Link" do
|
||||
response.headers["Location"].should eq(link)
|
||||
|
||||
# Verify that the click was recorded
|
||||
updated_test_link = get_test_link(test_link.id)
|
||||
updated_test_link = get_test_link(test_link.id.not_nil!)
|
||||
updated_test_link.clicks.size.should eq(test_link.clicks.size + 1)
|
||||
|
||||
# Verify click details
|
||||
@@ -137,14 +135,16 @@ describe "App::Controllers::Link" do
|
||||
test_user = create_test_user()
|
||||
|
||||
test_link = create_test_link(test_user, link)
|
||||
serialized_link = App::Serializers::Link.new(test_link)
|
||||
|
||||
# Add utm_source parameter
|
||||
get("#{serialized_link.refer}?utm_source=email_campaign")
|
||||
user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0"
|
||||
get("/#{test_link.slug}?utm_source=email_campaign", headers: HTTP::Headers{
|
||||
"User-Agent" => user_agent
|
||||
})
|
||||
|
||||
Fiber.yield
|
||||
sleep 0.2.seconds # Wait for async click creation
|
||||
|
||||
updated_test_link = get_test_link(test_link.id)
|
||||
updated_test_link = get_test_link(test_link.id.not_nil!)
|
||||
latest_click = updated_test_link.clicks.last
|
||||
latest_click.referer.should eq("email_campaign")
|
||||
end
|
||||
@@ -152,7 +152,7 @@ describe "App::Controllers::Link" do
|
||||
it "should return 404 - link does not exist" do
|
||||
test_user = create_test_user()
|
||||
|
||||
get("https://localhost:4001/R4kj2")
|
||||
get("/R4kj2")
|
||||
|
||||
expected = {"error" => "Resource not found"}.to_json
|
||||
response.status_code.should eq(404)
|
||||
@@ -171,7 +171,7 @@ describe "App::Controllers::Link" do
|
||||
|
||||
get("/api/links", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
|
||||
|
||||
parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body)
|
||||
parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body)
|
||||
|
||||
# Check that each link is in the response data
|
||||
origins = parsed_response["data"].as(Array).map { |link| link["origin"] }
|
||||
@@ -191,7 +191,7 @@ describe "App::Controllers::Link" do
|
||||
|
||||
get("/api/links?limit=2", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
|
||||
|
||||
parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body)
|
||||
parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body)
|
||||
parsed_response["data"].as(Array).size.should eq(2)
|
||||
parsed_response["pagination"].as(Hash)["has_more"].should be_true
|
||||
parsed_response["pagination"].as(Hash)["next"].should_not be_nil
|
||||
@@ -206,12 +206,12 @@ describe "App::Controllers::Link" do
|
||||
|
||||
# Get first page
|
||||
get("/api/links?limit=2", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
|
||||
first_page = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body)
|
||||
first_page = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body)
|
||||
cursor = first_page["pagination"].as(Hash)["next"]
|
||||
|
||||
# Get second page using cursor
|
||||
get("/api/links?limit=2&cursor=#{cursor}", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
|
||||
second_page = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body)
|
||||
second_page = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body)
|
||||
|
||||
# Ensure different links are returned
|
||||
first_page_ids = first_page["data"].as(Array).map { |link| link["id"] }
|
||||
@@ -234,7 +234,7 @@ describe "App::Controllers::Link" do
|
||||
|
||||
get("/api/links", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
|
||||
|
||||
parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body)
|
||||
parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body)
|
||||
parsed_response["data"].as(Array).size.should eq(3)
|
||||
|
||||
origins = parsed_response["data"].as(Array).map { |link| link["origin"] }
|
||||
@@ -265,7 +265,7 @@ describe "App::Controllers::Link" do
|
||||
|
||||
get("/api/links/#{test_link.id}", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
|
||||
|
||||
parsed_response = Hash(String, Hash(String, String | Int64 | Array(Hash(String, String)))).from_json(response.body)
|
||||
parsed_response = Hash(String, Hash(String, String | Int64 | Array(Hash(String, String | Int64)))).from_json(response.body)
|
||||
parsed_response["data"]["origin"].should eq(link)
|
||||
parsed_response["data"]["clicks"].as(Array).size.should eq(100)
|
||||
end
|
||||
@@ -273,7 +273,7 @@ describe "App::Controllers::Link" do
|
||||
it "should return 404 - link does not exist" do
|
||||
test_user = create_test_user()
|
||||
|
||||
get("/api/links/1", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
|
||||
get("/api/links/999999", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
|
||||
|
||||
expected = {"error" => "Resource not found"}.to_json
|
||||
response.status_code.should eq(404)
|
||||
@@ -301,7 +301,7 @@ describe "App::Controllers::Link" do
|
||||
|
||||
get("/api/links/#{test_link.id}/clicks", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
|
||||
|
||||
parsed_response = Hash(String, Array(Hash(String, String)) | Hash(String, Bool | String?)).from_json(response.body)
|
||||
parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body)
|
||||
parsed_response["data"].as(Array).size.should eq(5)
|
||||
parsed_response["pagination"].as(Hash)["has_more"].should be_false
|
||||
end
|
||||
@@ -317,7 +317,7 @@ describe "App::Controllers::Link" do
|
||||
|
||||
get("/api/links/#{test_link.id}/clicks?limit=3", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
|
||||
|
||||
parsed_response = Hash(String, Array(Hash(String, String)) | Hash(String, Bool | String?)).from_json(response.body)
|
||||
parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body)
|
||||
parsed_response["data"].as(Array).size.should eq(3)
|
||||
parsed_response["pagination"].as(Hash)["has_more"].should be_true
|
||||
parsed_response["pagination"].as(Hash)["next"].should_not be_nil
|
||||
@@ -334,12 +334,12 @@ describe "App::Controllers::Link" do
|
||||
|
||||
# Get first page
|
||||
get("/api/links/#{test_link.id}/clicks?limit=3", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
|
||||
first_page = Hash(String, Array(Hash(String, String)) | Hash(String, Bool | String?)).from_json(response.body)
|
||||
first_page = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body)
|
||||
cursor = first_page["pagination"].as(Hash)["next"]
|
||||
|
||||
# Get second page using cursor
|
||||
get("/api/links/#{test_link.id}/clicks?limit=3&cursor=#{cursor}", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
|
||||
second_page = Hash(String, Array(Hash(String, String)) | Hash(String, Bool | String?)).from_json(response.body)
|
||||
second_page = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body)
|
||||
|
||||
# Ensure different clicks are returned
|
||||
first_page_ids = first_page["data"].as(Array).map { |click| click["id"] }
|
||||
@@ -352,7 +352,7 @@ describe "App::Controllers::Link" do
|
||||
it "should return 404 - link does not exist" do
|
||||
test_user = create_test_user()
|
||||
|
||||
get("/api/links/nonexistent_id/clicks", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
|
||||
get("/api/links/999999/clicks", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
|
||||
|
||||
expected = {"error" => "Resource not found"}.to_json
|
||||
response.status_code.should eq(404)
|
||||
@@ -381,7 +381,7 @@ describe "App::Controllers::Link" do
|
||||
body: payload.to_json
|
||||
)
|
||||
|
||||
parsed_response = Hash(String, Hash(String, String | Int64 | Array(Hash(String, String)))).from_json(response.body)
|
||||
parsed_response = Hash(String, Hash(String, String | Int64 | Array(Hash(String, String | Int64)))).from_json(response.body)
|
||||
parsed_response["data"]["origin"].should eq(payload["url"])
|
||||
end
|
||||
|
||||
@@ -390,7 +390,7 @@ describe "App::Controllers::Link" do
|
||||
|
||||
payload = {"url" => "https://kagi.com.co"}
|
||||
put(
|
||||
"/api/links/1",
|
||||
"/api/links/999999",
|
||||
headers: HTTP::Headers{"Content-Type" => "application/json", "X-Api-Key" => test_user.api_key.to_s},
|
||||
body: payload.to_json
|
||||
)
|
||||
@@ -428,7 +428,7 @@ describe "App::Controllers::Link" do
|
||||
it "should return 404 - link does not exist" do
|
||||
test_user = create_test_user()
|
||||
|
||||
delete("/api/links/1", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
|
||||
delete("/api/links/999999", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
|
||||
|
||||
expected = {"error" => "Resource not found"}.to_json
|
||||
response.status_code.should eq(404)
|
||||
|
||||
+9
-8
@@ -32,12 +32,12 @@ def create_test_user
|
||||
raise "Test user creation failed #{error_messages}"
|
||||
end
|
||||
|
||||
user
|
||||
changeset.instance
|
||||
end
|
||||
|
||||
def create_test_link(user, url)
|
||||
link = App::Models::Link.new
|
||||
link.slug = App::Services::SlugService.shorten_url(url, user.id)
|
||||
link.slug = App::Services::SlugService.shorten_url(url, user.id.not_nil!)
|
||||
link.url = url
|
||||
link.user = user
|
||||
|
||||
@@ -47,9 +47,10 @@ def create_test_link(user, url)
|
||||
raise "Test link creation failed: #{error_messages}"
|
||||
end
|
||||
|
||||
link.clicks = [] of App::Models::Click
|
||||
inserted_link = changeset.instance
|
||||
inserted_link.clicks = [] of App::Models::Click
|
||||
|
||||
link
|
||||
inserted_link
|
||||
end
|
||||
|
||||
def create_test_click(link)
|
||||
@@ -61,17 +62,17 @@ def create_test_click(link)
|
||||
click.country = "US"
|
||||
click.created_at = Time.utc
|
||||
click.link = link
|
||||
click.link_id = link.id
|
||||
click.link_id = link.id.not_nil!
|
||||
|
||||
changeset = App::Lib::Database.insert(click)
|
||||
unless changeset.valid?
|
||||
error_messages = changeset.errors.map { |error| "#{error}" }.join(", ")
|
||||
raise "Test click creation failed: #{error_messages}"
|
||||
end
|
||||
click
|
||||
changeset.instance
|
||||
end
|
||||
|
||||
def get_test_link(link_id: Int64)
|
||||
def get_test_link(link_id : Int64)
|
||||
query = App::Lib::Database::Query.where(id: link_id).limit(1)
|
||||
link = App::Lib::Database.all(App::Models::Link, query, preload: [:clicks]).first?
|
||||
|
||||
@@ -80,6 +81,6 @@ def get_test_link(link_id: Int64)
|
||||
link
|
||||
end
|
||||
|
||||
def delete_test_link(link_id: Int64)
|
||||
def delete_test_link(link_id : Int64)
|
||||
App::Lib::Database.raw_exec("DELETE FROM links WHERE id = (?)", link_id) # tempfix: Database.delete does not work
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user