diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml new file mode 100644 index 0000000..38000d7 --- /dev/null +++ b/.github/workflows/deploy-docs.yml @@ -0,0 +1,33 @@ +name: Deploy API Documentation + +on: + push: + branches: + - master + paths: + - 'docs/openapi.yaml' + - '.github/workflows/deploy-docs.yml' + workflow_dispatch: + +permissions: + contents: write + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Generate Swagger UI + uses: Legion2/swagger-ui-action@v1 + with: + output: swagger-ui + spec-file: docs/openapi.yaml + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Deploy to GitHub Pages + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: swagger-ui diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f43023c..8220828 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -8,12 +8,9 @@ on: types: [published] jobs: - build-platforms: - name: Build Platforms + build-and-push: + name: Build and Push Multi-Platform runs-on: ubuntu-latest - strategy: - matrix: - platform: [linux/amd64, linux/arm64] permissions: contents: read packages: write @@ -22,8 +19,11 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to Docker Hub uses: docker/login-action@v3 @@ -40,45 +40,12 @@ jobs: echo "version=latest" >> $GITHUB_OUTPUT fi - - name: Build and push platform image + - name: Build and push multi-platform image uses: docker/build-push-action@v5 - env: - CRYSTAL_WORKERS: ${{ matrix.platform == 'linux/amd64' && 4 || 2 }} with: context: . - platforms: ${{ matrix.platform }} + platforms: linux/amd64,linux/arm64 push: true - tags: | - sjdonado/bit:${{ steps.version.outputs.version }}-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }} - build-args: | - TARGETARCH=${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }} + tags: sjdonado/bit:${{ steps.version.outputs.version }} cache-from: type=gha cache-to: type=gha,mode=max - - create-manifest: - name: Create Manifest - runs-on: ubuntu-latest - needs: build-platforms - permissions: - packages: write - steps: - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Determine version - id: version - run: | - if [ "${{ github.event_name }}" = "release" ]; then - echo "version=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT - else - echo "version=latest" >> $GITHUB_OUTPUT - fi - - - name: Create manifest - run: | - docker buildx imagetools create \ - -t sjdonado/bit:${{ steps.version.outputs.version }} \ - sjdonado/bit:${{ steps.version.outputs.version }}-{amd64,arm64} diff --git a/.github/workflows/update-parsers.yml b/.github/workflows/update-parsers.yml new file mode 100644 index 0000000..528dd40 --- /dev/null +++ b/.github/workflows/update-parsers.yml @@ -0,0 +1,63 @@ +name: Update Parsers + +on: + schedule: + # Run every two weeks on Sunday at 00:00 UTC (1st and 3rd Sunday of each month) + - cron: '0 0 1-7,15-21 * 0' + workflow_dispatch: # Allow manual trigger + +jobs: + update-parsers: + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Crystal + uses: crystal-lang/install-crystal@v1 + + - name: Install dependencies + run: shards install + + - name: Build CLI + run: crystal build scripts/cli.cr -o cli + + - name: Update parsers + run: ./cli --update-parsers + + - name: Check for changes + id: changes + run: | + if [ -n "$(git status --porcelain)" ]; then + echo "has_changes=true" >> $GITHUB_OUTPUT + else + echo "has_changes=false" >> $GITHUB_OUTPUT + fi + + - name: Create Pull Request + if: steps.changes.outputs.has_changes == 'true' + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: 'chore: update parsers' + title: 'chore: Update UA regexes and GeoLite2 database' + body: | + ## Automated Parser Update + + This PR updates the following data files: + - User Agent parsing regexes + - GeoLite2 database + + **Triggered by**: Scheduled workflow (runs every 2 weeks) + **Date**: ${{ github.event.repository.updated_at }} + + Please review the changes and merge if everything looks good. + branch: automated/update-parsers + delete-branch: true + labels: | + dependencies + automated diff --git a/.gitignore b/.gitignore index 050e0f0..15eabab 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ /sqlite/ .env.production -resource_usage.* +*.log +bit diff --git a/Dockerfile b/Dockerfile index c318b76..c761686 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,41 +1,37 @@ -FROM debian:bookworm-slim AS build +FROM alpine:edge AS build -ARG TARGETARCH ENV ENV=production WORKDIR /usr/src/app -RUN apt-get update && apt-get install -y \ - curl \ - gnupg \ - ca-certificates \ - && mkdir -p /etc/apt/keyrings \ - && curl -fsSL https://packagecloud.io/84codes/crystal/gpgkey | gpg --dearmor > /etc/apt/trusted.gpg.d/84codes_crystal.gpg \ - && echo "deb [signed-by=/etc/apt/trusted.gpg.d/84codes_crystal.gpg] https://packagecloud.io/84codes/crystal/debian/ bookworm main" > /etc/apt/sources.list.d/84codes_crystal.list - -RUN apt-get update && apt-get install -y \ +RUN apk add --no-cache \ crystal \ - libssl-dev \ - libyaml-dev \ - libsqlite3-dev \ + shards \ + openssl-dev \ + yaml-dev \ + sqlite-dev \ libevent-dev \ - && rm -rf /var/lib/apt/lists/* + tzdata COPY . . RUN shards install --production RUN shards build --release --no-debug --progress --stats -FROM debian:bookworm-slim AS runtime +FROM alpine:latest AS runtime ENV ENV=production WORKDIR /usr/src/app -RUN apt-get update && apt-get install -y \ - libssl3 \ - libyaml-0-2 \ - sqlite3 \ - libevent-2.1-7 \ - && rm -rf /var/lib/apt/lists/* +RUN apk add --no-cache \ + gc-dev \ + pcre2 \ + libevent \ + sqlite-libs \ + openssl \ + yaml \ + gmp \ + libgcc \ + tzdata RUN mkdir -p sqlite diff --git a/README.md b/README.md index 081f97a..549168c 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ [![Docker Stars](https://img.shields.io/docker/stars/sjdonado/bit.svg)](https://hub.docker.com/r/sjdonado/bit) [![Docker Image Size](https://img.shields.io/docker/image-size/sjdonado/bit/latest)](https://hub.docker.com/r/sjdonado/bit) -Lightweight URL shortener (API-only) with minimal resource requirements. Avg memory consumption **30MiB**, avg CPU load 20%. +Lightweight URL shortener (API-only) with minimal resource requirements. -Highly performant: **7.9k req/sec**, latency 15.8ms (100k requests using 125 connections, [benchmark](docs/SETUP.md#benchmark)). +Highly performant: **11k req/sec**, latency 11ms, 40MiB avg memory usage (100k requests using 125 connections, [benchmark](docs/SETUP.md#benchmark)). Self-hosted: [Dokku](docs/SETUP.md#dokku), [Docker Compose](docs/SETUP.md#docker-compose). @@ -20,7 +20,7 @@ It is feature-complete by design: simple and reliable without unnecessary bloat. ## Recommented requirements - 100MB disk space -- 70MiB RAM +- 100MiB RAM - x86_64 or ARM64 ## Documentation diff --git a/app/controllers/click.cr b/app/controllers/click.cr index b855510..0354159 100644 --- a/app/controllers/click.cr +++ b/app/controllers/click.cr @@ -4,90 +4,6 @@ module App::Controllers include App::Lib include App::Services - # Buffered channel to hold click data - @@click_channel = Channel(NamedTuple( - link_id: Int64, - remote_address: String, - user_agent: String?, - referer: String - )).new(1024) - - @@processor_started = begin - spawn do - batch = [] of NamedTuple( - link_id: Int64, - remote_address: String, - user_agent: String?, - referer: String - ) - - batch_size = 32 - min_batch = 32 - max_batch = 512 - scaling_factor = 1.5 - - loop do - select - when click_data = @@click_channel.receive - batch << click_data - - if batch.size >= batch_size - process_click_batch(batch) - batch.clear - # Increase batch size when reaching capacity - batch_size = (batch_size * scaling_factor).to_i.clamp(min_batch, max_batch) - end - when timeout(1.seconds) - unless batch.empty? - process_click_batch(batch) - batch.clear - batch_size = (batch_size / scaling_factor).to_i.clamp(min_batch, max_batch) - else - # Reset to default if idle - batch_size = min_batch unless batch_size == min_batch - end - end - end - end - true - end - - private def self.process_click_batch(batch) - clicks = [] of App::Models::Click - - batch.each do |click_data| - begin - client_ip = IpLookup.ip_from_address(click_data[:remote_address]) - family, _, _, os = UserAgent.parse(click_data[:user_agent] || "") - - click = App::Models::Click.new - click.link_id = click_data[:link_id] - click.country = client_ip ? IpLookup.country(client_ip) : nil - click.user_agent = click_data[:user_agent] - click.browser = family - click.os = os.try &.[0] # OS family - click.referer = click_data[:referer] - - clicks << click - rescue ex - Log.error { "Click data processing error: #{ex.message}" } - end - end - - # Batch insert clicks if any were successfully processed - unless clicks.empty? - begin - multi = Crecto::Multi.new - clicks.each do |click| - multi.insert(click) - end - Database.transaction(multi) - rescue ex - Log.error { "Batch click insertion error: #{ex.message}" } - end - end - end - def self.redirect_handler ->(env : HTTP::Server::Context) { link_id, url = Database.raw_query("SELECT id, url FROM links WHERE slug = (?) LIMIT 1", env.params.url["slug"]) do |result| @@ -96,21 +12,32 @@ module App::Controllers remote_address = env.request.headers["Cf-Connecting-Ip"]? || env.request.remote_address.to_s + # Send redirect immediately env.response.status_code = 301 env.response.headers.add("Location", url) env.response.headers.add("X-Forwarded-For", remote_address) + if user_agent = env.request.headers["User-Agent"]? + env.response.headers.add("User-Agent", user_agent) + end - begin - @@click_channel.send({ - link_id: link_id, - remote_address: remote_address, - user_agent: env.request.headers["User-Agent"]?, - referer: env.request.headers["Referer"]?.try { |r| URI.parse(r).host rescue r } || env.params.query["utm_source"]? || "Direct" - }) - rescue Channel::ClosedError - Log.error { "Click channel closed" } - rescue ex - Log.error { "Error queuing click: #{ex.message}" } + # non-blocking click proccessing + spawn do + begin + client_ip = IpLookup.ip_from_address(remote_address) + family, _, _, os = UserAgent.parse(env.request.headers["User-Agent"]? || "") + + click = App::Models::Click.new + click.link_id = link_id + click.country = client_ip ? IpLookup.country(client_ip) : nil + click.user_agent = env.request.headers["User-Agent"]? + click.browser = family + click.os = os.try &.[0] + click.referer = env.request.headers["Referer"]?.try { |r| URI.parse(r).host rescue r } || env.params.query["utm_source"]? || "Direct" + + Database.insert(click) + rescue ex + Log.error { "Click tracking error: #{ex.message}" } + end end } end diff --git a/app/controllers/link.cr b/app/controllers/link.cr index b8f7efb..024903b 100644 --- a/app/controllers/link.cr +++ b/app/controllers/link.cr @@ -145,7 +145,7 @@ module App::Controllers "data" => items.map { |item| yield item }, "pagination" => { "has_more" => has_more, - "next_cursor" => next_cursor + "next" => next_cursor } }) end diff --git a/app/lib/controller.cr b/app/lib/controller.cr index e721583..47be15d 100644 --- a/app/lib/controller.cr +++ b/app/lib/controller.cr @@ -23,7 +23,7 @@ module App::Lib missing_fields = required_fields.reject { |field| json_params.has_key?(field) } unless missing_fields.empty? - error_message = missing_fields.join(", ") + " required" + error_message = "#{missing_fields.first}: Required field" raise App::BadRequestException.new(@env, error_message) end diff --git a/app/services/cli.cr b/app/services/cli.cr index 4105ad6..edb097c 100644 --- a/app/services/cli.cr +++ b/app/services/cli.cr @@ -73,7 +73,7 @@ module App::Services::Cli end end - def self.download_geolite_db + def self.update_geolite_db puts "Downloading GeoLite2 Country database..." FileUtils.mkdir_p("data") diff --git a/benchmark.cr b/benchmark.cr deleted file mode 100755 index aa46bcd..0000000 --- a/benchmark.cr +++ /dev/null @@ -1,257 +0,0 @@ -#!/usr/bin/env crystal - -require "http/client" -require "json" -require "file_utils" - -# Configuration variables -SERVER_URL = "http://localhost:4000" -API_URL = "#{SERVER_URL}/api/links" -API_KEY = "secure_api_key_1" -NUMBER_OF_REQUESTS = 100000 - -CONTAINER_NAME = "bit" -STATS_FILE = "resource_usage.txt" - -class ResourceMonitor - def initialize(@container_name : String) - @running = false - @stats = [] of {timestamp: Time, cpu: Float64, memory: Float64} - end - - def start - @running = true - @stats.clear - - # Initialize stats file with header - File.write(STATS_FILE, "Timestamp\tCPU(%)\tMemory(MiB)\n") - - spawn do - while @running - if stat = capture_stats - # Append each measurement directly to the file - File.open(STATS_FILE, "a") do |file| - file.puts "#{stat[:timestamp].to_unix}\t#{stat[:cpu]}\t#{stat[:memory]}" - end - @stats << stat - end - end - end - end - - def stop - @running = false - end - - def avg_stats - return {cpu: 0.0, memory: 0.0} if @stats.empty? - - total_cpu = 0.0 - total_memory = 0.0 - @stats.each do |stat| - total_cpu += stat[:cpu] - total_memory += stat[:memory] - end - - { - cpu: total_cpu / @stats.size, - memory: total_memory / @stats.size - } - end - - private def capture_stats - output = IO::Memory.new - process = Process.run( - "docker", ["stats", "--no-stream", "--format", "{{.CPUPerc}},{{.MemUsage}}", @container_name], - output: output - ) - - if process.success? - line = output.to_s.strip - parts = line.split(",") - if parts.size == 2 - cpu_part = parts[0].gsub("%", "").to_f - - # Extract the memory value properly by removing the "MiB" suffix - mem_string = parts[1].split.first - mem_part = mem_string.gsub(/[A-Za-z]+$/, "").to_f - - return {timestamp: Time.utc, cpu: cpu_part, memory: mem_part} - end - end - nil - end -end - -def check_dependencies - {"docker", "jq", "bombardier"}.each do |cmd| - process = Process.run("which", [cmd], output: Process::Redirect::Close) - unless process.success? - puts "Error: #{cmd} is not installed. Please install it to proceed." - exit(1) - end - end -end - -def setup_containers - puts "Setting up..." - - process = Process.run("docker", ["compose", "up", "-d"]) - unless process.success? - puts "Failed to start Docker containers." - exit(1) - end - - puts "Waiting for the application to be ready..." - until begin - HTTP::Client.get("#{SERVER_URL}/api/ping").success? - rescue - false - end - sleep 1.seconds - end - - puts "Seeding the database..." - process = Process.run("docker", ["compose", "exec", "app", "sh", "-c", "sqlite3 ./sqlite/data.db < ./db/seed.sql"]) - - unless process.success? - puts "Error on seeding database" - exit(1) - end - - puts "Checking seed results..." - until begin - HTTP::Client.get( - "#{API_URL}?limit=1", - headers: HTTP::Headers{"X-Api-Key" => API_KEY} - ).success? - rescue - false - end - sleep 2.seconds - end -end - -def run_benchmark - puts "Fetching all created links from /api/links..." - - response = HTTP::Client.get( - "#{API_URL}?limit=10000", - headers: HTTP::Headers{"X-Api-Key" => API_KEY} - ) - - sleep 2.seconds - unless response.success? - puts "Failed to fetch links. Status: #{response.status_code}" - exit(1) - end - - data = JSON.parse(response.body) - links = data["data"].as_a.map { |link| link["refer"].as_s } - - random_link = links.sample - puts "Selected link for benchmarking: #{random_link}" - - puts "Starting benchmark with Bombardier..." - - sleep 2.seconds - process = Process.new( - "bombardier", - ["-n", NUMBER_OF_REQUESTS.to_s, "-l", "--disableKeepAlives", random_link], - output: Process::Redirect::Inherit, - error: Process::Redirect::Inherit - ) - - status = process.wait - - if status.success? - puts "Benchmark completed successfully." - else - puts "Bombardier failed with error code: #{status.exit_code}" - exit(1) - end -end - -def analyze_resource_usage - puts "Analyzing resource usage..." - - sleep 2.seconds - if File.exists?(STATS_FILE) - lines = File.read_lines(STATS_FILE) - # Skip header - lines = lines[1..-1] if lines.size > 0 - - if lines.size > 0 - total_cpu = 0.0 - total_memory = 0.0 - peak_cpu = 0.0 - peak_memory = 0.0 - - lines.each do |line| - fields = line.split("\t") - if fields.size >= 3 - begin - cpu = fields[1].to_f - memory = fields[2].to_f - - total_cpu += cpu - total_memory += memory - - # Track peaks in a single pass - peak_cpu = cpu if cpu > peak_cpu - peak_memory = memory if memory > peak_memory - rescue - # Skip invalid lines - end - end - end - - avg_cpu = total_cpu / lines.size - avg_memory = total_memory / lines.size - - stats_summary = <<-STATS - **** Resource Usage Statistics **** - Measurements: #{lines.size} - Average CPU Usage: #{avg_cpu.round(2)}% - Average Memory Usage: #{avg_memory.round(2)} MiB - Peak CPU Usage: #{peak_cpu.round(2)}% - Peak Memory Usage: #{peak_memory.round(2)} MiB - - STATS - - File.open(STATS_FILE, "a") do |file| - file.puts "\n" + stats_summary - end - - puts File.read(STATS_FILE) - else - puts "No resource usage data collected." - end - else - puts "Resource usage file not found." - end -end - -def cleanup - Process.run("docker", ["compose", "down"]) - puts "Cleanup completed. Resource usage data saved in #{STATS_FILE}" -end - -def main - check_dependencies - setup_containers - - monitor = ResourceMonitor.new(CONTAINER_NAME) - monitor.start - - begin - run_benchmark - - monitor.stop - analyze_resource_usage - ensure - cleanup - end -end - -main diff --git a/docs/API.md b/docs/API.md deleted file mode 100644 index 83a54fd..0000000 --- a/docs/API.md +++ /dev/null @@ -1,149 +0,0 @@ -# API Reference - -1. **Ping the API** - - - Endpoint: `GET /api/ping` - - Payload: None - - Response: 200 - - Response Example - ```json - { - "data": "pong" - } - ``` -2. **Redirect by Slug** - - - Endpoint: `GET /:slug` - - Payload: None - - Response: 301 - -3. **List All Links** - - - Endpoint: `GET /api/links` - - Headers: `X-Api-Key` - - Query Parameters: - - `limit` (optional): Number of results per page (default: 100) - - `cursor` (optional): Pagination cursor from previous response - - Response: 200 - - Response Example - ```json - { - "data": [ - { - "id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db", - "refer": "http://localhost:4000/3wP4BQ", - "origin": "https://monocuco.donado.co" - } - ], - "pagination": { - "has_more": true, - "next": "75e0a7f4-9c5e-1235-b546-eb9c5e40f7ac" - } - } - ``` - -4. **List link by ID** - - Endpoint: `GET /api/links/:id` - - Headers: `X-Api-Key` - - Payload: None - - Note: This endpoint returns up to 100 of the most recent clicks. For complete click history, use the `/api/links/:id/clicks` endpoint with pagination. - - Response: 200 - - Response Example - ```json - { - "data": { - "id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db", - "refer": "http://localhost:4000/3wP4BQ", - "origin": "https://monocuco.donado.co", - "clicks": [ - { - "id": "730e2202-58f9-478c-a24c-f1c561df6716", - "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0", - "country": "DE", - "browser": "Firefox", - "os": "Mac OS X", - "referer": "Direct", - "created_at": "2024-07-12T19:25:22Z" - } - ] - } - } - ``` - -5. **List Clicks for a Link** - - Endpoint: `GET /api/links/:id/clicks` - - Headers: `X-Api-Key` - - Query Parameters: - - `limit` (optional): Number of results per page (default: 100) - - `cursor` (optional): Pagination cursor from previous response - - Response: 200 - - Response Example - ```json - { - "data": [ - { - "id": "730e2202-58f9-478c-a24c-f1c561df6716", - "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0", - "country": "DE", - "browser": "Firefox", - "os": "Mac OS X", - "referer": "Direct", - "created_at": "2024-07-12T19:25:22Z" - } - ], - "pagination": { - "has_more": true, - "next": "629e3301-47f8-389b-b24c-f1c561df9825" - } - } - ``` - -6. **Create new link** - - Endpoint: `POST /api/links` - - Payload: - ```json - { - "url": "https://example.com" - } - ``` - - Headers: `X-Api-Key` - - Response: 201 - - Response Example: - ```json - { - "data": { - "id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db", - "refer": "http://localhost:4000/3wP4BQ", - "origin": "https://example.com", - "clicks": [] - } - } - ``` - -7. **Update an existing link by ID** - - Endpoint: `PUT /api/links/:id` - - Payload: - ```json - { - "url": "https://newexample.com" - } - ``` - - Headers: `X-Api-Key` - - Response: 200 - - Response Example: - ```json - { - "data": { - "id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db", - "refer": "http://localhost:4000/3wP4BQ", - "origin": "https://newexample.com", - "clicks": [] - } - } - ``` - -8. **Delete a link by ID** - - Endpoint: `DELETE /api/links/:id` - - Payload: None - - Headers: `X-Api-Key` - - Response: 204 diff --git a/docs/SETUP.md b/docs/SETUP.md index 0ca1e25..967e4c4 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -73,7 +73,7 @@ Recommended for lower latency communication (no host network traversal) ## Local Development ### Requirements -- Crystal 1.12+ +- Crystal 1.18+ - Shards package manager - SQLite3 @@ -109,59 +109,140 @@ ENV=test crystal spec ## Benchmark -- Colima: cpu 1, mem 1 -- SoC: Apple M3 Pro +### Run ``` -~/p/bit> colima start --cpu 1 --memory 1 -INFO[0000] starting colima -INFO[0000] runtime: docker -INFO[0001] starting ... context=vm -INFO[0076] provisioning ... context=docker -INFO[0077] starting ... context=docker -INFO[0077] done -~/p/bit> ./benchmark.cr -Setting up... -Waiting for the application to be ready... -Seeding the database... -Checking seed results... -Fetching all created links from /api/links... -Selected link for benchmarking: http://localhost:4000/slug2576 -Starting benchmark with Bombardier... -Bombarding http://localhost:4000/slug2576 with 100000 request(s) using 125 connection(s) - 100000 / 100000 [===================================================================================================================================================================] 100.00% 7795/s 12s +shards build --release --no-debug --progress --stats +shards run benchmark +``` + +### Output + +Chip: Apple M4 Pro. Memory: 24GB + +``` +1762075350 ~/p/bit> shards build --release --no-debug --progress --stats + shards run benchmark +Dependencies are satisfied +Building: bit +Parse: 00:00:00.000652375 ( 1.17MB) +Semantic (top level): 00:00:00.419246250 ( 163.45MB) +Semantic (new): 00:00:00.001636125 ( 163.45MB) +Semantic (type declarations): 00:00:00.019569792 ( 179.45MB) +Semantic (abstract def check): 00:00:00.009145125 ( 195.45MB) +Semantic (restrictions augmenter): 00:00:00.008421709 ( 195.45MB) +Semantic (ivars initializers): 00:00:00.019696584 ( 211.45MB) +Semantic (cvars initializers): 00:00:00.106829666 ( 211.50MB) +Semantic (main): 00:00:00.649298375 ( 499.88MB) +Semantic (cleanup): 00:00:00.000765250 ( 499.88MB) +Semantic (recursive struct check): 00:00:00.000752250 ( 499.88MB) +Codegen (crystal): 00:00:00.521307417 ( 532.38MB) +Codegen (bc+obj): 00:00:00.143842542 ( 532.38MB) +Codegen (linking): 00:00:00.236228750 ( 532.38MB) + +Macro runs: + - /opt/homebrew/Cellar/crystal/1.18.2/share/crystal/src/ecr/process.cr: reused previous compilation (00:00:00.003593375) + +Codegen (bc+obj): + - all previous .o files were reused +Building: cli +Parse: 00:00:00.000053291 ( 1.17MB) +Semantic (top level): 00:00:00.323534167 ( 163.45MB) +Semantic (new): 00:00:00.001705500 ( 163.45MB) +Semantic (type declarations): 00:00:00.018311958 ( 179.45MB) +Semantic (abstract def check): 00:00:00.007766750 ( 195.45MB) +Semantic (restrictions augmenter): 00:00:00.005686667 ( 195.45MB) +Semantic (ivars initializers): 00:00:00.011239792 ( 211.45MB) +Semantic (cvars initializers): 00:00:00.100870833 ( 211.50MB) +Semantic (main): 00:00:00.285426750 ( 371.62MB) +Semantic (cleanup): 00:00:00.000369875 ( 371.62MB) +Semantic (recursive struct check): 00:00:00.000570917 ( 371.62MB) +Codegen (crystal): 00:00:00.317534875 ( 387.88MB) +Codegen (bc+obj): 00:00:00.097321417 ( 387.88MB) +Codegen (linking): 00:00:00.095931000 ( 387.88MB) + +Codegen (bc+obj): + - all previous .o files were reused +Building: benchmark +Parse: 00:00:00.000228500 ( 1.17MB) +Semantic (top level): 00:00:00.242174458 ( 147.78MB) +Semantic (new): 00:00:00.000863333 ( 147.78MB) +Semantic (type declarations): 00:00:00.011527792 ( 147.78MB) +Semantic (abstract def check): 00:00:00.031242333 ( 147.78MB) +Semantic (restrictions augmenter): 00:00:00.003593583 ( 147.78MB) +Semantic (ivars initializers): 00:00:00.006753667 ( 147.78MB) +Semantic (cvars initializers): 00:00:00.028373834 ( 195.78MB) +Semantic (main): 00:00:00.152039542 ( 243.83MB) +Semantic (cleanup): 00:00:00.000249084 ( 243.83MB) +Semantic (recursive struct check): 00:00:00.000460417 ( 243.83MB) +Codegen (crystal): 00:00:00.075461000 ( 259.83MB) +Codegen (bc+obj): 00:00:04.834914333 ( 259.83MB) +Codegen (linking): 00:00:00.119920416 ( 259.83MB) + +Codegen (bc+obj): + - no previous .o files were reused +Dependencies are satisfied +Building: benchmark +Executing: benchmark +Cleaning up benchmark database... +Deleted existing database: ./sqlite/data.benchmark.db +Database cleanup completed. +Running database migrations... +Migrating db, current version: 0, target: 20250319192003 +OK 20240512214223_create_links.sql +OK 20240512225208_add_slug_index_to_links.sql +OK 20240513115731_create_users.sql +OK 20240513130054_add_api_key_index_to_users.sql +OK 20240711224103_create_clicks.sql +OK 20240714215409_update_slug_size_links.sql +OK 20250316102350_add_country_to_clicks.sql +OK 20250316111734_replace_unkwown_with_null.sql +OK 20250318072657_replace_slug_index_with_covering_index.sql +OK 20250319192003_convert_all_tables_text_ids_to_integer.sql +Migrations completed successfully. +Seeding benchmark database... +Database seeded successfully. +Starting application: ./bit... +Application output will be saved to: app_output.log +Application started with PID: 11638 +Using database: ./sqlite/data.benchmark.db +Checking if server is ready at http://localhost:4001... +.Server is ready! +Fetching links from API... +Selected link: http://localhost:4001/slug9391 + +Starting benchmark with 100000 requests... +Bombarding http://localhost:4001/slug9391 with 100000 request(s) using 125 connection(s) + 100000 / 100000 [============================================================================] 100.00% 11078/s 9s Done! Statistics Avg Stdev Max - Reqs/sec 7900.70 7570.15 29263.59 - Latency 15.89ms 10.22ms 67.32ms + Reqs/sec 11427.28 8889.68 30270.91 + Latency 11.02ms 6.55ms 53.91ms Latency Distribution - 50% 4.82ms - 75% 9.24ms - 90% 51.61ms - 95% 52.74ms - 99% 55.07ms + 50% 1.85ms + 75% 5.37ms + 90% 39.36ms + 95% 39.87ms + 99% 42.66ms HTTP codes: 1xx - 0, 2xx - 0, 3xx - 100000, 4xx - 0, 5xx - 0 others - 0 - Throughput: 2.14MB/s + Throughput: 3.08MB/s + Benchmark completed successfully. -Analyzing resource usage... -Timestamp CPU(%) Memory(MiB) -1742763202 0.01 44.83 -1742763204 0.01 44.78 -1742763206 91.53 68.23 -1742763208 92.03 68.17 -1742763210 91.0 68.09 -1742763212 92.73 68.38 -1742763214 92.17 67.66 -1742763216 91.1 67.69 -1742763218 2.93 67.04 **** Resource Usage Statistics **** - Measurements: 9 - Average CPU Usage: 61.5% - Average Memory Usage: 62.76 MiB - Peak CPU Usage: 92.73% - Peak Memory Usage: 68.38 MiB -Cleanup completed. Resource usage data saved in resource_usage.txt + Measurements: 12 + Average CPU Usage: 71.5% + Average Memory Usage: 39.8 MiB + Peak CPU Usage: 100.0% + Peak Memory Usage: 53.41 MiB + +**** Files Generated **** + Resource stats: resource_usage.log + Application log: app_output.log + Database: ./sqlite/data.benchmark.db + +Stopping application... +Application stopped. ``` diff --git a/docs/openapi.yaml b/docs/openapi.yaml new file mode 100644 index 0000000..8a3d70f --- /dev/null +++ b/docs/openapi.yaml @@ -0,0 +1,496 @@ +openapi: 3.1.0 +info: + title: Bit - URL Shortener API + description: A high-performance URL shortener service with click tracking and analytics + version: 1.0.0 + contact: + name: API Support + url: https://github.com/sjdonado/bit + +servers: + - url: http://localhost:4000 + description: Development server + - url: http://localhost:4001 + description: Benchmark server + +security: + - ApiKeyAuth: [] + +paths: + /api/ping: + get: + summary: Ping the API + description: Health check endpoint to verify the API is running + operationId: ping + tags: + - Health + security: [] + responses: + '200': + description: API is healthy + content: + application/json: + schema: + type: object + properties: + data: + type: string + example: pong + + /{slug}: + get: + summary: Redirect by slug + description: Redirects to the original URL and tracks the click asynchronously + operationId: redirectBySlug + tags: + - Redirects + security: [] + parameters: + - name: slug + in: path + required: true + description: The short URL slug + schema: + type: string + example: 3wP4BQ + - name: utm_source + in: query + required: false + description: UTM source parameter for tracking + schema: + type: string + example: email_campaign + responses: + '301': + description: Redirect to original URL + headers: + Location: + description: The original URL + schema: + type: string + example: https://example.com + X-Forwarded-For: + description: Client IP address + schema: + type: string + User-Agent: + description: User agent string + schema: + type: string + '404': + description: Link not found + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + + /api/links: + get: + summary: List all links + description: Retrieve all links for the authenticated user with pagination support + operationId: listLinks + tags: + - Links + parameters: + - name: limit + in: query + description: Number of results per page + schema: + type: integer + default: 100 + minimum: 1 + maximum: 1000 + - name: cursor + in: query + description: Pagination cursor from previous response + schema: + type: string + responses: + '200': + description: List of links + content: + application/json: + schema: + type: object + properties: + data: + type: array + items: + $ref: '#/components/schemas/LinkSummary' + pagination: + $ref: '#/components/schemas/Pagination' + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + + post: + summary: Create new link + description: Create a new shortened link + operationId: createLink + tags: + - Links + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - url + properties: + url: + type: string + format: uri + description: The URL to shorten + example: https://example.com + responses: + '201': + description: Link created successfully + content: + application/json: + schema: + type: object + properties: + data: + $ref: '#/components/schemas/Link' + '400': + description: Bad request - invalid URL or missing field + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + examples: + missingField: + value: + error: "url: Required field" + invalidUrl: + value: + errors: + url: + - is invalid + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + + /api/links/{id}: + get: + summary: Get link by ID + description: Retrieve a specific link with up to 100 most recent clicks. For complete click history, use /api/links/{id}/clicks + operationId: getLink + tags: + - Links + parameters: + - name: id + in: path + required: true + description: Link ID + schema: + type: integer + format: int64 + responses: + '200': + description: Link details + content: + application/json: + schema: + type: object + properties: + data: + $ref: '#/components/schemas/Link' + '404': + description: Link not found + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + + put: + summary: Update link + description: Update the URL of an existing link + operationId: updateLink + tags: + - Links + parameters: + - name: id + in: path + required: true + description: Link ID + schema: + type: integer + format: int64 + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - url + properties: + url: + type: string + format: uri + description: The new URL + example: https://newexample.com + responses: + '200': + description: Link updated successfully + content: + application/json: + schema: + type: object + properties: + data: + $ref: '#/components/schemas/Link' + '400': + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '403': + description: Forbidden - link belongs to another user + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '404': + description: Link not found + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + + delete: + summary: Delete link + description: Delete a link and all its associated clicks + operationId: deleteLink + tags: + - Links + parameters: + - name: id + in: path + required: true + description: Link ID + schema: + type: integer + format: int64 + responses: + '204': + description: Link deleted successfully + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '403': + description: Forbidden - link belongs to another user + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '404': + description: Link not found + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + + /api/links/{id}/clicks: + get: + summary: List clicks for a link + description: Retrieve all clicks for a specific link with pagination support + operationId: listClicks + tags: + - Clicks + parameters: + - name: id + in: path + required: true + description: Link ID + schema: + type: integer + format: int64 + - name: limit + in: query + description: Number of results per page + schema: + type: integer + default: 100 + minimum: 1 + maximum: 1000 + - name: cursor + in: query + description: Pagination cursor from previous response + schema: + type: string + responses: + '200': + description: List of clicks + content: + application/json: + schema: + type: object + properties: + data: + type: array + items: + $ref: '#/components/schemas/Click' + pagination: + $ref: '#/components/schemas/Pagination' + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '404': + description: Link not found + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + +components: + securitySchemes: + ApiKeyAuth: + type: apiKey + in: header + name: X-Api-Key + description: API key for authentication + + schemas: + LinkSummary: + type: object + properties: + id: + type: integer + format: int64 + description: Unique link identifier + example: 1 + refer: + type: string + format: uri + description: The shortened URL + example: http://localhost:4000/3wP4BQ + origin: + type: string + format: uri + description: The original URL + example: https://monocuco.donado.co + + Link: + allOf: + - $ref: '#/components/schemas/LinkSummary' + - type: object + properties: + clicks: + type: array + description: Array of click records (up to 100 most recent) + items: + $ref: '#/components/schemas/Click' + + Click: + type: object + properties: + id: + type: integer + format: int64 + description: Unique click identifier + example: 1 + user_agent: + type: string + description: User agent string + example: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0 + country: + type: string + nullable: true + description: Country code (ISO 3166-1 alpha-2) + example: US + browser: + type: string + nullable: true + description: Browser name + example: Firefox + os: + type: string + nullable: true + description: Operating system + example: Mac OS X + referer: + type: string + nullable: true + description: Referer domain or utm_source + example: Direct + created_at: + type: string + format: date-time + description: Click timestamp + example: 2024-07-12T19:25:22Z + + Pagination: + type: object + properties: + has_more: + type: boolean + description: Whether there are more results + example: true + next: + type: integer + format: int64 + nullable: true + description: Cursor for next page (link/click ID) + example: 12 + + Error: + type: object + properties: + error: + type: string + description: Error message + example: Resource not found + required: + - error + + ValidationErrors: + type: object + properties: + errors: + type: object + additionalProperties: + type: array + items: + type: string + description: Field-level validation errors + example: + url: + - is invalid + +tags: + - name: Health + description: Health check endpoints + - name: Redirects + description: URL redirection and click tracking + - name: Links + description: Link management operations + - name: Clicks + description: Click analytics and tracking diff --git a/scripts/benchmark.cr b/scripts/benchmark.cr new file mode 100644 index 0000000..e7e6faf --- /dev/null +++ b/scripts/benchmark.cr @@ -0,0 +1,341 @@ +#!/usr/bin/env crystal + +require "http/client" +require "json" + +PORT = "4001" +APP_URL = "http://localhost:#{PORT}" +API_URL = "#{APP_URL}/api/links" +API_KEY = "secure_api_key_1" +NUMBER_OF_REQUESTS = 100000 + +APP_COMMAND = "./bit" +APP_ARGS = [] of String +STATS_FILE = "resource_usage.log" +APP_LOG_FILE = "app_output.log" + +DATABASE_URL = "sqlite3://./sqlite/data.benchmark.db?journal_mode=wal&synchronous=normal&foreign_keys=true" +DATABASE_FILE = "./sqlite/data.benchmark.db" + +class ResourceMonitor + def initialize(@pid : Int32) + @running = false + @stats = [] of {timestamp: Time, cpu: Float64, memory: Float64} + end + + def start + @running = true + @stats.clear + File.write(STATS_FILE, "Timestamp\tCPU(%)\tMemory(MiB)\n") + + spawn do + while @running + if stat = capture_stats + File.open(STATS_FILE, "a") do |file| + file.puts "#{stat[:timestamp].to_unix}\t#{stat[:cpu]}\t#{stat[:memory]}" + end + @stats << stat + end + sleep 1.seconds + end + end + end + + def stop + @running = false + sleep 1.seconds + end + + private def capture_stats + output = IO::Memory.new + process = Process.run( + "ps", ["-p", @pid.to_s, "-o", "%cpu,%mem,rss"], + output: output + ) + + if process.success? + lines = output.to_s.strip.split("\n") + if lines.size >= 2 + data_line = lines[1].strip.split + if data_line.size >= 3 + cpu = data_line[0].to_f + # RSS is in KB on macOS, convert to MiB + memory_kb = data_line[2].to_f + memory_mib = memory_kb / 1024.0 + + return {timestamp: Time.utc, cpu: cpu, memory: memory_mib} + end + end + end + nil + end + + def print_summary + return if @stats.empty? + + total_cpu = 0.0 + total_memory = 0.0 + peak_cpu = 0.0 + peak_memory = 0.0 + + @stats.each do |stat| + total_cpu += stat[:cpu] + total_memory += stat[:memory] + peak_cpu = stat[:cpu] if stat[:cpu] > peak_cpu + peak_memory = stat[:memory] if stat[:memory] > peak_memory + end + + avg_cpu = total_cpu / @stats.size + avg_memory = total_memory / @stats.size + + summary = <<-STATS + + **** Resource Usage Statistics **** + Measurements: #{@stats.size} + Average CPU Usage: #{avg_cpu.round(2)}% + Average Memory Usage: #{avg_memory.round(2)} MiB + Peak CPU Usage: #{peak_cpu.round(2)}% + Peak Memory Usage: #{peak_memory.round(2)} MiB + + STATS + + File.open(STATS_FILE, "a") do |file| + file.puts summary + end + + puts summary + end +end + +def start_application : Process + puts "Starting application: #{APP_COMMAND}..." + puts "Application output will be saved to: #{APP_LOG_FILE}" + + log_file = File.open(APP_LOG_FILE, "w") + + process = Process.new( + APP_COMMAND, + APP_ARGS, + output: log_file, + error: log_file, + env: { + "DATABASE_URL" => DATABASE_URL, + "APP_URL" => APP_URL, + "PORT" => PORT, + } + ) + + puts "Application started with PID: #{process.pid}" + puts "Using database: #{DATABASE_FILE}" + process +end + +def stop_application(process : Process) + puts "\nStopping application..." + process.signal(Signal::TERM) + + # Give it a few seconds to shut down gracefully + sleep 3.seconds + + # Force kill if still running + begin + process.signal(Signal::KILL) + rescue + # Process already terminated + end + + puts "Application stopped." +end + +def check_dependencies + {"bombardier", "sqlite3", "micrate"}.each do |cmd| + process = Process.run("which", [cmd], output: Process::Redirect::Close) + unless process.success? + puts "Error: #{cmd} is not installed. Please install it to proceed." + case cmd + when "bombardier" + puts " brew install bombardier" + when "sqlite3" + puts " brew install sqlite3" + when "micrate" + puts " shards install" + end + exit(1) + end + end +end + +def wait_for_server + puts "Checking if server is ready at #{APP_URL}..." + + 30.times do + begin + if HTTP::Client.get("#{APP_URL}/api/ping").success? + puts "Server is ready!" + return + end + rescue + # Server not ready yet + end + sleep 1.seconds + print "." + end + + puts "\nError: Server is not responding. Please start your application first." + exit(1) +end + +def run_benchmark + puts "Fetching links from API..." + + response = HTTP::Client.get( + "#{API_URL}?limit=10000", + headers: HTTP::Headers{"X-Api-Key" => API_KEY} + ) + + unless response.success? + puts "Failed to fetch links. Status: #{response.status_code}" + puts "Make sure your server is running and the API key is correct." + exit(1) + end + + data = JSON.parse(response.body) + links = data["data"].as_a.map { |link| link["refer"].as_s } + + if links.empty? + puts "No links found. Please seed your database first." + exit(1) + end + + random_link = links.sample + puts "Selected link: #{random_link}" + puts "\nStarting benchmark with #{NUMBER_OF_REQUESTS} requests..." + + sleep 2.seconds + + process = Process.new( + "bombardier", + ["-n", NUMBER_OF_REQUESTS.to_s, "-l", "--disableKeepAlives", random_link], + output: Process::Redirect::Inherit, + error: Process::Redirect::Inherit + ) + + status = process.wait + + if status.success? + puts "\nBenchmark completed successfully." + else + puts "\nBombardier failed with error code: #{status.exit_code}" + exit(1) + end +end + +def cleanup_database + puts "Cleaning up benchmark database..." + + if File.exists?(DATABASE_FILE) + File.delete(DATABASE_FILE) + puts "Deleted existing database: #{DATABASE_FILE}" + end + + # Also remove WAL and SHM files if they exist + ["#{DATABASE_FILE}-wal", "#{DATABASE_FILE}-shm"].each do |file| + if File.exists?(file) + File.delete(file) + puts "Deleted: #{file}" + end + end + + # Ensure sqlite directory exists + Dir.mkdir_p("./sqlite") + puts "Database cleanup completed." +end + +def run_migrations + puts "Running database migrations..." + + process = Process.run("which", ["micrate"], output: Process::Redirect::Close) + unless process.success? + puts "Error: micrate is not installed. Please install it to proceed." + puts " shards install" + exit(1) + end + + process = Process.run( + "micrate", + ["up"], + env: {"DATABASE_URL" => DATABASE_URL}, + output: Process::Redirect::Inherit, + error: Process::Redirect::Inherit + ) + + if process.success? + puts "Migrations completed successfully." + else + puts "Error: Migrations failed." + exit(1) + end +end + +def seed_database + puts "Seeding benchmark database..." + + unless File.exists?("./db/seed.sql") + puts "Warning: ./db/seed.sql not found. Skipping database seeding." + return + end + + unless File.exists?(DATABASE_FILE) + puts "Warning: #{DATABASE_FILE} not found. Database may not be initialized." + end + + process = Process.run( + "sqlite3", + [DATABASE_FILE], + input: File.open("./db/seed.sql"), + output: Process::Redirect::Inherit, + error: Process::Redirect::Inherit + ) + + if process.success? + puts "Database seeded successfully." + else + puts "Warning: Database seeding failed. Continuing anyway..." + end +end + +def main + check_dependencies + + # Setup benchmark database + cleanup_database + run_migrations + seed_database + + app_process = start_application + + begin + wait_for_server + + # Give it a moment to settle + sleep 2.seconds + + monitor = ResourceMonitor.new(app_process.pid.to_i32) + monitor.start + + run_benchmark + + monitor.stop + monitor.print_summary + + puts "\n**** Files Generated ****" + puts " Resource stats: #{STATS_FILE}" + puts " Application log: #{APP_LOG_FILE}" + puts " Database: #{DATABASE_FILE}" + ensure + # Always stop the application + stop_application(app_process) + end +end + +main diff --git a/scripts/cli.cr b/scripts/cli.cr index f49d0bc..a38e32f 100644 --- a/scripts/cli.cr +++ b/scripts/cli.cr @@ -21,7 +21,7 @@ OptionParser.parse do |parser| parser.on("--update-parsers", "Download UA regexes and/or GeoLite2 database") do puts "=== Starting data files update ===" App::Services::Cli.update_uap_regexes - App::Services::Cli.download_geolite_db + App::Services::Cli.update_geolite_db puts "=== Data files updated successfully ===" exit end diff --git a/shard.lock b/shard.lock index 36d2245..218196b 100644 --- a/shard.lock +++ b/shard.lock @@ -2,11 +2,11 @@ version: 2.0 shards: backtracer: git: https://github.com/sija/backtracer.cr.git - version: 1.2.2 + version: 1.2.4 crecto: git: https://github.com/fridgerator/crecto.git - version: 0.12.1 + version: 0.14.0 db: git: https://github.com/crystal-lang/crystal-db.git @@ -18,7 +18,7 @@ shards: exception_page: git: https://github.com/crystal-loot/exception_page.git - version: 0.4.1 + version: 0.5.0 ipaddress: git: https://github.com/sija/ipaddress.cr.git @@ -26,7 +26,7 @@ shards: kemal: git: https://github.com/kemalcr/kemal.git - version: 1.5.0 + version: 1.7.3 maxminddb: git: https://github.com/delef/maxminddb.cr.git diff --git a/shard.yml b/shard.yml index 529cb82..f311cc4 100644 --- a/shard.yml +++ b/shard.yml @@ -1,18 +1,21 @@ name: bit -version: 1.5.3 +version: 1.6.0 authors: - - Juan Rodriguez + - Juan Rodriguez Donado <@sjdonado> targets: bit: main: bit.cr cli: main: scripts/cli.cr + benchmark: + main: scripts/benchmark.cr dependencies: kemal: github: kemalcr/kemal + version: 1.7.3 sqlite3: github: crystal-lang/crystal-sqlite3 crecto: @@ -29,6 +32,6 @@ development_dependencies: spec-kemal: github: kemalcr/spec-kemal -crystal: ">= 1.12.1" +crystal: ">= 1.18.2" license: MIT diff --git a/spec/integration/link_spec.cr b/spec/integration/link_spec.cr index aeaef60..6ec0cb3 100644 --- a/spec/integration/link_spec.cr +++ b/spec/integration/link_spec.cr @@ -15,7 +15,7 @@ describe "App::Controllers::Link" do body: payload.to_json ) - parsed_response = Hash(String, Hash(String, String | Int64 | Array(Hash(String, String)))).from_json(response.body) + parsed_response = Hash(String, Hash(String, String | Int64 | Array(Hash(String, String | Int64)))).from_json(response.body) parsed_response["data"]["origin"].should eq(payload["url"]) end @@ -87,11 +87,10 @@ describe "App::Controllers::Link" do test_user = create_test_user() test_link = create_test_link(test_user, link) - serialized_link = App::Serializers::Link.new(test_link) user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0" - get(serialized_link.refer, headers: HTTP::Headers{ + get("/#{test_link.slug}", headers: HTTP::Headers{ "X-Api-Key" => test_user.api_key.to_s, "User-Agent" => user_agent }) @@ -106,12 +105,11 @@ describe "App::Controllers::Link" do test_user = create_test_user() test_link = create_test_link(test_user, link) - serialized_link = App::Serializers::Link.new(test_link) user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0" referer = "https://example.com/page" - get(serialized_link.refer, headers: HTTP::Headers{ + get("/#{test_link.slug}", headers: HTTP::Headers{ "User-Agent" => user_agent, "Referer" => referer }) @@ -121,7 +119,7 @@ describe "App::Controllers::Link" do response.headers["Location"].should eq(link) # Verify that the click was recorded - updated_test_link = get_test_link(test_link.id) + updated_test_link = get_test_link(test_link.id.not_nil!) updated_test_link.clicks.size.should eq(test_link.clicks.size + 1) # Verify click details @@ -137,14 +135,16 @@ describe "App::Controllers::Link" do test_user = create_test_user() test_link = create_test_link(test_user, link) - serialized_link = App::Serializers::Link.new(test_link) # Add utm_source parameter - get("#{serialized_link.refer}?utm_source=email_campaign") + user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0" + get("/#{test_link.slug}?utm_source=email_campaign", headers: HTTP::Headers{ + "User-Agent" => user_agent + }) - Fiber.yield + sleep 0.2.seconds # Wait for async click creation - updated_test_link = get_test_link(test_link.id) + updated_test_link = get_test_link(test_link.id.not_nil!) latest_click = updated_test_link.clicks.last latest_click.referer.should eq("email_campaign") end @@ -152,7 +152,7 @@ describe "App::Controllers::Link" do it "should return 404 - link does not exist" do test_user = create_test_user() - get("https://localhost:4001/R4kj2") + get("/R4kj2") expected = {"error" => "Resource not found"}.to_json response.status_code.should eq(404) @@ -171,7 +171,7 @@ describe "App::Controllers::Link" do get("/api/links", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) - parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body) + parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body) # Check that each link is in the response data origins = parsed_response["data"].as(Array).map { |link| link["origin"] } @@ -191,7 +191,7 @@ describe "App::Controllers::Link" do get("/api/links?limit=2", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) - parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body) + parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body) parsed_response["data"].as(Array).size.should eq(2) parsed_response["pagination"].as(Hash)["has_more"].should be_true parsed_response["pagination"].as(Hash)["next"].should_not be_nil @@ -206,12 +206,12 @@ describe "App::Controllers::Link" do # Get first page get("/api/links?limit=2", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) - first_page = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body) + first_page = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body) cursor = first_page["pagination"].as(Hash)["next"] # Get second page using cursor get("/api/links?limit=2&cursor=#{cursor}", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) - second_page = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body) + second_page = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body) # Ensure different links are returned first_page_ids = first_page["data"].as(Array).map { |link| link["id"] } @@ -234,7 +234,7 @@ describe "App::Controllers::Link" do get("/api/links", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) - parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String?)).from_json(response.body) + parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body) parsed_response["data"].as(Array).size.should eq(3) origins = parsed_response["data"].as(Array).map { |link| link["origin"] } @@ -265,7 +265,7 @@ describe "App::Controllers::Link" do get("/api/links/#{test_link.id}", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) - parsed_response = Hash(String, Hash(String, String | Int64 | Array(Hash(String, String)))).from_json(response.body) + parsed_response = Hash(String, Hash(String, String | Int64 | Array(Hash(String, String | Int64)))).from_json(response.body) parsed_response["data"]["origin"].should eq(link) parsed_response["data"]["clicks"].as(Array).size.should eq(100) end @@ -273,7 +273,7 @@ describe "App::Controllers::Link" do it "should return 404 - link does not exist" do test_user = create_test_user() - get("/api/links/1", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) + get("/api/links/999999", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) expected = {"error" => "Resource not found"}.to_json response.status_code.should eq(404) @@ -301,7 +301,7 @@ describe "App::Controllers::Link" do get("/api/links/#{test_link.id}/clicks", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) - parsed_response = Hash(String, Array(Hash(String, String)) | Hash(String, Bool | String?)).from_json(response.body) + parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body) parsed_response["data"].as(Array).size.should eq(5) parsed_response["pagination"].as(Hash)["has_more"].should be_false end @@ -317,7 +317,7 @@ describe "App::Controllers::Link" do get("/api/links/#{test_link.id}/clicks?limit=3", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) - parsed_response = Hash(String, Array(Hash(String, String)) | Hash(String, Bool | String?)).from_json(response.body) + parsed_response = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body) parsed_response["data"].as(Array).size.should eq(3) parsed_response["pagination"].as(Hash)["has_more"].should be_true parsed_response["pagination"].as(Hash)["next"].should_not be_nil @@ -334,12 +334,12 @@ describe "App::Controllers::Link" do # Get first page get("/api/links/#{test_link.id}/clicks?limit=3", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) - first_page = Hash(String, Array(Hash(String, String)) | Hash(String, Bool | String?)).from_json(response.body) + first_page = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body) cursor = first_page["pagination"].as(Hash)["next"] # Get second page using cursor get("/api/links/#{test_link.id}/clicks?limit=3&cursor=#{cursor}", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) - second_page = Hash(String, Array(Hash(String, String)) | Hash(String, Bool | String?)).from_json(response.body) + second_page = Hash(String, Array(Hash(String, String | Int64)) | Hash(String, Bool | String? | Int64?)).from_json(response.body) # Ensure different clicks are returned first_page_ids = first_page["data"].as(Array).map { |click| click["id"] } @@ -352,7 +352,7 @@ describe "App::Controllers::Link" do it "should return 404 - link does not exist" do test_user = create_test_user() - get("/api/links/nonexistent_id/clicks", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) + get("/api/links/999999/clicks", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) expected = {"error" => "Resource not found"}.to_json response.status_code.should eq(404) @@ -381,7 +381,7 @@ describe "App::Controllers::Link" do body: payload.to_json ) - parsed_response = Hash(String, Hash(String, String | Int64 | Array(Hash(String, String)))).from_json(response.body) + parsed_response = Hash(String, Hash(String, String | Int64 | Array(Hash(String, String | Int64)))).from_json(response.body) parsed_response["data"]["origin"].should eq(payload["url"]) end @@ -390,7 +390,7 @@ describe "App::Controllers::Link" do payload = {"url" => "https://kagi.com.co"} put( - "/api/links/1", + "/api/links/999999", headers: HTTP::Headers{"Content-Type" => "application/json", "X-Api-Key" => test_user.api_key.to_s}, body: payload.to_json ) @@ -428,7 +428,7 @@ describe "App::Controllers::Link" do it "should return 404 - link does not exist" do test_user = create_test_user() - delete("/api/links/1", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) + delete("/api/links/999999", headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s}) expected = {"error" => "Resource not found"}.to_json response.status_code.should eq(404) diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index 416614d..6294214 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -32,12 +32,12 @@ def create_test_user raise "Test user creation failed #{error_messages}" end - user + changeset.instance end def create_test_link(user, url) link = App::Models::Link.new - link.slug = App::Services::SlugService.shorten_url(url, user.id) + link.slug = App::Services::SlugService.shorten_url(url, user.id.not_nil!) link.url = url link.user = user @@ -47,9 +47,10 @@ def create_test_link(user, url) raise "Test link creation failed: #{error_messages}" end - link.clicks = [] of App::Models::Click + inserted_link = changeset.instance + inserted_link.clicks = [] of App::Models::Click - link + inserted_link end def create_test_click(link) @@ -61,17 +62,17 @@ def create_test_click(link) click.country = "US" click.created_at = Time.utc click.link = link - click.link_id = link.id + click.link_id = link.id.not_nil! changeset = App::Lib::Database.insert(click) unless changeset.valid? error_messages = changeset.errors.map { |error| "#{error}" }.join(", ") raise "Test click creation failed: #{error_messages}" end - click + changeset.instance end -def get_test_link(link_id: Int64) +def get_test_link(link_id : Int64) query = App::Lib::Database::Query.where(id: link_id).limit(1) link = App::Lib::Database.all(App::Models::Link, query, preload: [:clicks]).first? @@ -80,6 +81,6 @@ def get_test_link(link_id: Int64) link end -def delete_test_link(link_id: Int64) +def delete_test_link(link_id : Int64) App::Lib::Database.raw_exec("DELETE FROM links WHERE id = (?)", link_id) # tempfix: Database.delete does not work end