chore: update API docs

Chore/documentation

.github/workflows/publish.yml

@@ -1,4 +1,4 @@
-name: Publish Docker image
+name: Publish Docker images
 
 on:
   push:
@@ -8,21 +8,20 @@ on:
     types: [published]
 
 jobs:
-  push_to_registry:
-    name: Push Docker image to Docker Hub
+  build-platforms:
+    name: Build Platforms
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        platform: [linux/amd64, linux/arm64]
     permissions:
-      packages: write
       contents: read
-      attestations: write
-      id-token: write
+      packages: write
+
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
@@ -32,33 +31,54 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Extract version from shard.yml
-        id: extract_version
+      - name: Determine version
+        id: version
         run: |
-          VERSION=$(grep '^version:' shard.yml | cut -d ' ' -f 2)
-          echo "RELEASE_TAG=$VERSION" >> $GITHUB_ENV
-
-      - name: Set tags
-        id: set_tags
-        run: |
-          if [[ "${{ github.event_name }}" == "release" ]]; then
-            echo "TAGS=latest,${{ env.RELEASE_TAG }}" >> $GITHUB_ENV
+          if [ "${{ github.event_name }}" = "release" ]; then
+            echo "version=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT
           else
-            echo "TAGS=latest" >> $GITHUB_ENV
+            echo "version=latest" >> $GITHUB_OUTPUT
           fi
 
-      - name: Build and push image
-        id: push
+      - name: Build and push platform image
         uses: docker/build-push-action@v5
+        env:
+          CRYSTAL_WORKERS: ${{ matrix.platform == 'linux/amd64' && 4 || 2 }}
         with:
           context: .
+          platforms: ${{ matrix.platform }}
           push: true
-          platforms: linux/amd64,linux/arm64
-          tags: sjdonado/bit:${{ env.TAGS }}
+          tags: |
+            sjdonado/bit:${{ steps.version.outputs.version }}-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
+          build-args: |
+            TARGETARCH=${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
 
-      - name: Attest
-        uses: actions/attest-build-provenance@v1
-        id: attest
+  create-manifest:
+    name: Create Manifest
+    runs-on: ubuntu-latest
+    needs: build-platforms
+    permissions:
+      packages: write
+    steps:
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
         with:
-          subject-name: sjdonado/bit
-          subject-digest: ${{ steps.push.outputs.digest }}
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Determine version
+        id: version
+        run: |
+          if [ "${{ github.event_name }}" = "release" ]; then
+            echo "version=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT
+          else
+            echo "version=latest" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Create manifest
+        run: |
+          docker buildx imagetools create \
+            -t sjdonado/bit:${{ steps.version.outputs.version }} \
+            sjdonado/bit:${{ steps.version.outputs.version }}-{amd64,arm64}

.gitignore

@@ -1,4 +1,3 @@
-/docs/
 /lib/
 /bin/
 /.shards/

CODE_OF_CONDUCT.md

@@ -0,0 +1,132 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official email address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+[INSERT CONTACT METHOD].
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+[https://www.contributor-covenant.org/version/2/0/code_of_conduct.html][v2.0].
+
+Community Impact Guidelines were inspired by 
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available 
+at [https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.0]: https://www.contributor-covenant.org/version/2/0/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

CONTRIBUTING.md

@@ -0,0 +1,75 @@
+# Contributing Guidelines
+
+We welcome contributions from the community! Please follow these guidelines to help maintain consistency and quality in the project.
+
+## Code of Conduct
+This project adheres to the [Contributor Covenant Code of Conduct](CODE_OF_CONDUCT.md). By participating, you agree to uphold its terms.
+
+## How to Contribute
+
+### 1. Fork the Repository
+Click the "Fork" button at the top-right of the [repository page](https://github.com/sjdonado/bit).
+
+### 2. Clone Your Fork
+```bash
+git clone https://github.com/YOUR_USERNAME/bit.git
+cd bit
+```
+
+### 3. Create a Feature Branch
+```bash
+git checkout -b feat/your-feature-name
+```
+
+### or for bug fixes:
+
+```bash
+git checkout -b fix/issue-description
+```
+
+### 4. Develop Your Changes
+- Check [Local Development](docs/SETUP.md#local-development) guidelines
+- Ensure changes match the project scope
+- Write clear commit messages
+- Include tests for new functionality
+- Update documentation when applicable
+
+### 5. Commit Changes
+```bash
+git commit -am 'Add descriptive commit message'
+```
+
+### 6. Push to GitHub
+```bash
+git push origin your-branch-name
+```
+
+### 7. Create a Pull Request
+1. Go to the [original repository](https://github.com/sjdonado/bit)
+2. Click "New Pull Request"
+3. Select your fork and branch
+4. Add a clear description including:
+   - Purpose of changes
+   - Related issues (if applicable)
+   - Testing performed
+
+## Pull Request Guidelines
+- Keep PRs focused on a single feature/bugfix
+- Ensure all tests pass
+- Update documentation in the same PR
+- Use descriptive titles (e.g., "Add URL validation" not "Update code")
+- Reference related issues using #issue-number
+
+## Reporting Issues
+When opening an issue, please include:
+1. Description of the problem
+2. Steps to reproduce
+3. Expected vs actual behavior
+4. Environment details (OS, Crystal version, etc)
+
+For feature requests:
+- Explain the problem you're trying to solve
+- Suggest potential implementations
+
+## License
+By contributing, you agree that your contributions will be licensed under the [license](LICENSE).

Dockerfile

@@ -1,32 +1,41 @@
-FROM alpine:edge AS build
+FROM debian:bookworm-slim AS build
 
+ARG TARGETARCH
 ENV ENV=production
 WORKDIR /usr/src/app
 
-RUN apk update && apk add --no-cache \
+RUN apt-get update && apt-get install -y \
+    curl \
+    gnupg \
+    ca-certificates \
+    && mkdir -p /etc/apt/keyrings \
+    && curl -fsSL https://packagecloud.io/84codes/crystal/gpgkey | gpg --dearmor > /etc/apt/trusted.gpg.d/84codes_crystal.gpg \
+    && echo "deb [signed-by=/etc/apt/trusted.gpg.d/84codes_crystal.gpg] https://packagecloud.io/84codes/crystal/debian/ bookworm main" > /etc/apt/sources.list.d/84codes_crystal.list
+
+RUN apt-get update && apt-get install -y \
     crystal \
-    shards \
-    yaml-dev \
-    sqlite-dev \
-    openssl-dev
+    libssl-dev \
+    libyaml-dev \
+    libsqlite3-dev \
+    libevent-dev \
+    && rm -rf /var/lib/apt/lists/*
 
 COPY . .
 
-RUN shards install
-RUN shards build --release --no-debug
+RUN shards install --production
+RUN shards build --release --no-debug --progress --stats
 
-FROM alpine:edge AS runtime
+FROM debian:bookworm-slim AS runtime
 
 ENV ENV=production
 WORKDIR /usr/src/app
 
-RUN apk add --no-cache \
-    gc \
-    pcre2 \
-    libevent \
-    yaml \
-    sqlite-libs \
-    openssl
+RUN apt-get update && apt-get install -y \
+    libssl3 \
+    libyaml-0-2 \
+    libsqlite3-0 \
+    libevent-2.1-7 \
+    && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p sqlite
 

README.md

@@ -1,296 +1,28 @@
-[![Docker Pulls](https://img.shields.io/docker/pulls/sjdonado/bit.svg)](https://hub.docker.com/repository/docker/sjdonado/bit)
-[![Docker Stars](https://img.shields.io/docker/stars/sjdonado/bit.svg)](https://hub.docker.com/repository/docker/sjdonado/bit)
-[![Docker Image Size](https://img.shields.io/docker/image-size/sjdonado/bit/latest)](https://hub.docker.com/repository/docker/sjdonado/bit)
+[![Docker Pulls](https://img.shields.io/docker/pulls/sjdonado/bit.svg)](https://hub.docker.com/r/sjdonado/bit)
+[![Docker Stars](https://img.shields.io/docker/stars/sjdonado/bit.svg)](https://hub.docker.com/r/sjdonado/bit)
+[![Docker Image Size](https://img.shields.io/docker/image-size/sjdonado/bit/latest)](https://hub.docker.com/r/sjdonado/bit)
 
-## API Endpoints
+Lightweight URL shortener service with minimal resource requirements. Average memory consumption is **20MB RAM** with container disk space under **50MB**.
 
-1. **Ping the API**
+Bit is highly performant, achieving over 1.8K requests per second with an average latency of 68ms. For detailed benchmark results, see [benchmark](docs/SETUP.md#benchmark).
 
-   - Endpoint: `GET /api/ping`
-   - Payload: None
-   - Response Example
-     ```json
-     {
-       "message": "pong"
-     }
-     ```
+Images available on [Docker Hub](https://hub.docker.com/r/sjdonado/bit/tags).
 
-2. **Retrieve a link by its slug**
+## Why Bit?
+It is feature-complete by design. Its strength lies in simplicity, a reliable URL shortener without unnecessary bloat. Bug fixes will continue, but new features aren't planned.
 
-   - Endpoint: `GET /:slug`
-   - Headers: `X-Api-Key`
-   - Payload: None
-   - Response Example
-     ```json
-     {
-       "data": {
-         "id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db",
-         "refer": "http://localhost:4000/3wP4BQ",
-         "origin": "https://monocuco.donado.co",
-         "clicks": [
-           {
-             "id": "730e2202-58f9-478c-a24c-f1c561df6716",
-             "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0",
-             "language": "en-US",
-             "browser": "Firefox",
-             "os": "Mac OS X",
-             "source": "Unknown",
-             "created_at": "2024-07-12T19:25:22Z"
-           }
-         ]
-       }
-     }
-     ```
+- Minimal tracking setup: Country, browser, os, referer. No cookies or persistent tracking mechanisms are used beyond what's available from a basic client's request.
+- Flexible request forwarding system passes client context (IP, user-agent) to destinations via standard X-Forwarded-For and X-Forwarded-User-Agent headers, enabling advanced tracking and integration capabilities when needed.
+- Multiple users are supported via API key authentication. Create, list and delete via the [CLI](docs/SETUP.md#cli).
 
-3. **Retrieve all links**
+## Minimum Requirements
+- 50MB disk space
+- 50MB RAM (20MB avg usage)
+- x86_64 or ARM64 architecture
 
-   - Endpoint: `GET /api/links`
-   - Headers: `X-Api-Key`
-   - Payload: None
-   - Response Example
-     ```json
-     {
-       "data": [
-         {
-           "id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db",
-           "refer": "http://localhost:4000/3wP4BQ",
-           "origin": "https://monocuco.donado.co",
-           "clicks": [
-             {
-               "id": "730e2202-58f9-478c-a24c-f1c561df6716",
-               "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0",
-               "language": "en-US",
-               "browser": "Firefox",
-               "os": "Mac OS X",
-               "source": "Unknown",
-               "created_at": "2024-07-12T19:25:22Z"
-             }
-           ]
-         }
-       ]
-     }
-     ```
-
-4. **Retrieve a link by its ID**
-
-   - Endpoint: `GET /api/links/:id`
-   - Headers: `X-Api-Key`
-   - Payload: None
-   - Response Example
-     ```json
-     {
-       "data": {
-         "id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db",
-         "refer": "http://localhost:4000/3wP4BQ",
-         "origin": "https://monocuco.donado.co",
-         "clicks": [
-           {
-             "id": "730e2202-58f9-478c-a24c-f1c561df6716",
-             "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0",
-             "language": "en-US",
-             "browser": "Firefox",
-             "os": "Mac OS X",
-             "source": "Unknown",
-             "created_at": "2024-07-12T19:25:22Z"
-           }
-         ]
-       }
-     }
-     ```
-
-5. **Create a new link**
-
-   - Endpoint\*\*: `POST /api/links`
-   - Payload:
-     ```json
-     {
-       "url": "https://example.com"
-     }
-     ```
-   - Headers: `X-Api-Key`
-   - Response Example:
-     ```json
-     {
-       "data": {
-         "id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db",
-         "refer": "http://localhost:4000/3wP4BQ",
-         "origin": "https://monocuco.donado.co/test",
-         "clicks": []
-       }
-     }
-     ```
-
-6. **Update an existing link by its ID**
-
-   - Endpoint: `PUT /api/links/:id`
-   - Payload:
-     ```json
-     {
-       "url": "https://newexample.com"
-     }
-     ```
-   - Headers: `X-Api-Key`
-   - Response Example:
-     ```json
-     {
-       "data": {
-         "id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db",
-         "refer": "http://localhost:4000/3wP4BQ",
-         "origin": "https://newexample.com",
-         "clicks": []
-       }
-     }
-     ```
-
-7. **Delete a link by its ID**
-
-   - Endpoint: `DELETE /api/links/:id`
-   - Payload: None
-   - Headers: `X-Api-Key`
-   - Response Example:
-     ```json
-     {
-       "message": "Link deleted"
-     }
-     ```
-
-## Self-hosted
-
-### Run via docker-compose
-
-```bash
-docker-compose up
-
-# Optional: Generate an api key
-# docker-compose exec -it app cli --create-user=Admin
-```
-
-### Run via docker cli
-
-```bash
-docker run \
-    --name bit \
-    -p 4000:4000 \
-    -e ENV="production" \
-    -e DATABASE_URL="sqlite3://./sqlite/data.db?journal_mode=wal&synchronous=normal&foreign_keys=true" \
-    -e APP_URL="http://localhost:4000" \
-    -e ADMIN_NAME="Admin" \
-    -e ADMIN_API_KEY=$(openssl rand -base64 32) \
-    sjdonado/bit
-
-# Optional: Generate an api key
-# docker exec -it bit cli --create-user=Admin
-```
-
-### Dokku
-
-```dockerfile
-FROM sjdonado/bit
-```
-
-```bash
-dokku apps:create bit
-
-dokku domains:set bit bit.donado.co
-dokku letsencrypt:enable bit
-
-dokku storage:ensure-directory bit-sqlite
-dokku storage:mount bit /var/lib/dokku/data/storage/bit-sqlite:/usr/src/app/sqlite/
-
-dokku config:set bit DATABASE_URL="sqlite3://./sqlite/data.db?journal_mode=wal&synchronous=normal&foreign_keys=true" APP_URL=https://bit.donado.co ADMIN_NAME=Admin ADMIN_API_KEY=$(openssl rand -base64 32)
-
-dokku ports:add bit http:80:4000
-dokku ports:add bit https:443:4000
-
-# Optional: Generate an api key
-# dokku run bit cli --create-user=Admin
-```
-
-## CLI
-
-```
-Usage: ./cli [options]
-Options:
-  --create-user=NAME  Create a new user with the given name
-  --list-users        List all users
-  --delete-user=USER_ID Delete a user by ID
-```
-
-## Benchmark
-
-```
-$ ./benchmark.sh
-Setting up...
-[+] Running 3/3
- ✔ Network bit_default       Created                                                                         0.0s
- ✔ Volume "bit_sqlite_data"  Created                                                                         0.0s
- ✔ Container bit             Started                                                                         0.1s
-Captured API Key: aHOCnZSuo2kOHy2mDa-iOA
-Waiting for the application to be ready...
-HTTP/1.1 200 OK
-Connection: keep-alive
-Content-Type: application/json
-Date: Sun, 27 Oct 2024 11:52:33 GMT
-Access-Control-Allow-Origin: *
-Access-Control-Allow-Methods: GET, POST, PUT, DELETE, OPTIONS
-Access-Control-Allow-Headers: Content-Type, Accept, Origin, X-Api-Key
-Content-Length: 13
-
-Starting resource usage monitoring...
-Creating 10000 short links with 100 conrurrent requests...
-Link creation complete: 10000 links created.
-Fetching all created links from /api/links...
-Selected link for benchmarking: http://localhost:4000/UaVZjA
-Starting benchmark with Bombardier...
-Bombarding http://localhost:4000/oEKLAg with 10000 request(s) using 100 connection(s)
- 10000 / 10000 [===============================================================================] 100.00% 830/s 12s
-Done!
-Statistics        Avg      Stdev        Max
-  Reqs/sec       853.89    1625.49    8942.54
-  Latency      118.48ms    11.52ms   142.58ms
-  HTTP codes:
-    1xx - 0, 2xx - 0, 3xx - 10000, 4xx - 0, 5xx - 0
-    others - 0
-  Throughput:   360.02KB/s
-Benchmark completed.
-Analyzing resource usage...
-**** Results ****
-Average CPU Usage: 40.68%
-Average Memory Usage: 28.62 MiB
-./benchmark.sh: line 135: 61567 Terminated: 15          monitor_resource_usage
-[+] Running 2/2
- ✔ Container bit        Removed                                                                                                                                                                                               10.1s
- ✔ Network bit_default  Removed  
-```
-
-## Development
-
-- Setup
-```bash
-brew tap amberframework/micrate
-brew install micrate
-```
-
-```bash
-shards run bit
-```
-
-- Generate the `X-Api-Key`
-
-```bash
-shards run cli -- --create-user=Admin
-```
-
-- Run tests
-
-```bash
-ENV=test crystal spec
-```
+## Documentation
+- [API Reference](docs/API.md)
+- [Advanced Setup](docs/SETUP.md)
 
 ## Contributing
-
-1. Fork it (<https://github.com/sjdonado/bit/fork>)
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create a new Pull Request
+Found an issue or have a suggestion? Please follow our [contribution guidelines](CONTRIBUTING.md).

app/controllers/link.cr

@@ -1,9 +1,11 @@
 require "uuid"
 require "user_agent_parser"
 
-UserAgent.load_regexes(File.read("data/regexes.yaml"))
-
 require "../lib/controller.cr"
+require "../lib/ip_lookup"
+
+UserAgent.load_regexes(File.read("data/uap_core_regexes.yaml"))
+IpLookup.load_mmdb("data/GeoLite2-Country.mmdb")
 
 module App::Controllers::Link
   class Create < App::Lib::BaseController
@@ -51,34 +53,40 @@ module App::Controllers::Link
       link = Database.get_by(Link, slug: slug)
       raise App::NotFoundException.new(env) if !link
 
+      remote_address = env.request.remote_address.try &.to_s
+      user_agent_str = env.request.headers["User-Agent"]? || "Unknown"
+
+      client_ip = IpLookup.extract_ip(remote_address) || "Unknown"
+
+      env.response.status_code = 301
+      env.response.headers["Location"] = link.url!
+
+      env.response.headers["X-Forwarded-For"] = client_ip
+      env.response.headers["X-Forwarded-User-Agent"] = user_agent_str
+
       spawn do
-        user_agent_str = env.request.headers["User-Agent"]? || "Unknown"
+        ip_lookup = client_ip != "Unknown" ? IpLookup.new(client_ip) : nil
+        country = ip_lookup.try &.country.try &.code
+
         user_agent = user_agent_str != "Unknown" ? UserAgent.new(user_agent_str) : nil
 
-        language_header = env.request.headers["Accept-Language"]? || "Unknown"
-        language = language_header.split(',').first.split(';').first
-
-        referer = env.request.headers["Referer"]?
+        source = env.params.query["utm_source"]? || "Direct"
+        referer_host = env.request.headers["Referer"]?.try { |r| begin URI.parse(r).host rescue r end } || source
 
         click = Click.new
         click.id = UUID.v4.to_s
         click.link = link
-        click.language = language
+        click.country = country
         click.user_agent = user_agent_str
-        click.browser = user_agent ? user_agent.family : "Unknown"
-        click.os = user_agent ? (user_agent.os.try &.family || "Unknown") : "Unknown"
-        click.source = referer ? URI.parse(referer).host : "Unknown"
+        click.browser = user_agent.try &.family
+        click.os = user_agent.try &.os.try &.family
+        click.referer = referer_host
 
         changeset = Database.insert(click)
         if changeset.errors.any?
           Log.error { "Logging click event failed: #{changeset.errors}" }
         end
       end
-
-      env.response.status_code = 301
-      env.response.headers["Location"] = link.url!
-      env.response.headers["Content-Type"] = "text/html"
-      env.response.print("Redirecting...")
     end
   end
 

app/controllers/ping.cr

@@ -3,7 +3,7 @@ require "../lib/controller.cr"
 module App::Controllers::Ping
   class Get < App::Lib::BaseController
     def call(env)
-      response = {"pong" => "ok"}
+      response = {"data" => "pong"}
       response.to_json
     end
   end

app/lib/ip_lookup.cr

@@ -0,0 +1,54 @@
+require "maxminddb"
+
+class IpLookup
+  @@instance : MaxMindDB::Reader? = nil
+
+  record Country, code : String? = nil, name : String? = nil
+
+  getter ip : String
+  getter country : Country?
+
+  def self.load_mmdb(mmdb_file_path : String)
+    @@instance = MaxMindDB.open(mmdb_file_path)
+  end
+
+  def initialize(ip_address : String)
+    @ip = ip_address
+    @country = nil
+
+    return if @@instance.nil? || ip_address == "Unknown" || ip_address.empty?
+
+    begin
+      lookup = @@instance.not_nil!.get(ip_address)
+
+      country_code = lookup["country"]?.try &.["iso_code"]?.try &.as_s
+      country_name = lookup["country"]?.try &.["names"]?.try &.["en"]?.try &.as_s
+
+      if country_code || country_name
+        @country = Country.new(
+          code: country_code,
+          name: country_name
+        )
+      end
+    rescue ex
+      # Silently handle lookup errors
+      Log.error { "IP lookup failed: #{ex.message}" }
+    end
+  end
+
+  def self.extract_ip(address_string : String?) : String?
+    return nil if address_string.nil?
+
+    if address_string.includes?('[') # IPv6 with port: [2001:db8::1]:8080
+      address_string.split(']').first.sub('[', '\'')
+    elsif address_string.includes?(':')
+      if address_string.count(':') > 1 # IPv6 without port
+        address_string
+      else # IPv4 with port: 192.168.1.1:8080
+        address_string.split(':').first
+      end
+    else # Address without port
+      address_string
+    end
+  end
+end

app/models/click.cr

@@ -5,14 +5,14 @@ module App::Models
     schema :clicks do
       field :id, String, primary_key: true
       field :user_agent, String
-      field :language, String
+      field :country, String
       field :browser, String
       field :os, String
-      field :source, String
+      field :referer, String
 
       belongs_to :link, Link
     end
 
-    validate_required [:user_agent, :language, :source]
+    validate_required [:user_agent, :referer]
   end
 end

app/serializers/click.cr

@@ -11,10 +11,10 @@ module App::Serializers
       builder.object do
         builder.field("id", @click.id)
         builder.field("user_agent", @click.user_agent)
-        builder.field("language", @click.language)
+        builder.field("country", @click.country)
         builder.field("browser", @click.browser)
         builder.field("os", @click.os)
-        builder.field("source", @click.source)
+        builder.field("referer", @click.referer)
         builder.field("created_at", @click.created_at)
       end
     end

app/services/cli.cr

@@ -1,3 +1,6 @@
+require "file_utils"
+require "http/client"
+
 require "../config/*"
 require "../lib/*"
 require "../models/*"
@@ -53,4 +56,86 @@ module App::Services::Cli
       puts "Admin setup skipped: Missing ADMIN_NAME or ADMIN_API_KEY environment variables."
     end
   end
+
+  def self.update_uap_regexes
+    puts "Downloading User-Agent Parser core regexes..."
+
+    FileUtils.mkdir_p("data")
+    url = "https://raw.githubusercontent.com/ua-parser/uap-core/master/regexes.yaml"
+    output_file = "data/uap_core_regexes.yaml"
+
+    begin
+      http_get_with_redirect(url) do |response|
+        File.write(output_file, response.body_io.gets_to_end)
+      end
+      puts "User-Agent regexes downloaded to #{output_file}"
+    rescue e
+      puts "Error: Failed to download UAP core regexes: #{e.message}"
+    end
+  end
+
+  def self.download_geolite_db
+    puts "Downloading GeoLite2 Country database..."
+
+    FileUtils.mkdir_p("data")
+    url = "https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-Country.mmdb"
+    output_file = "data/GeoLite2-Country.mmdb"
+
+    begin
+      File.open(output_file, "wb") do |file|
+        http_get_with_redirect(url) do |response|
+          IO.copy(response.body_io, file)
+        end
+      end
+      puts "GeoLite2 database downloaded to #{output_file}"
+    rescue e
+      puts "Error: Failed to download GeoLite2 database: #{e.message}"
+    end
+  end
+
+  private def self.http_get_with_redirect(url : String, max_redirects = 5)
+    redirects = 0
+
+    while redirects < max_redirects
+      uri = URI.parse(url)
+      client = HTTP::Client.new(uri)
+
+      success = false
+      follow_redirect = false
+      redirect_url = nil
+
+      begin
+        client.get(uri.request_target) do |response|
+          case response.status_code
+          when 200
+            yield response
+            success = true
+          when 301, 302
+            if new_location = response.headers["Location"]?
+              puts "Following redirect to: #{new_location}"
+              redirect_url = new_location
+              follow_redirect = true
+            else
+              raise "Received redirect status but no Location header"
+            end
+          else
+            raise "Failed request with status code: #{response.status_code}"
+          end
+        end
+      ensure
+        client.close
+      end
+
+      return if success
+
+      if follow_redirect && redirect_url
+        url = redirect_url
+        redirects += 1
+      else
+        break
+      end
+    end
+
+    raise "Too many redirects (#{max_redirects})"
+  end
 end

data/uap_core_regexes.yaml

@@ -148,11 +148,11 @@ user_agent_parsers:
     family_replacement: 'Pinterestbot'
 
   # Bots
-  - regex: '(CSimpleSpider|Cityreview Robot|CrawlDaddy|CrawlFire|Finderbots|Index crawler|Job Roboter|KiwiStatus Spider|Lijit Crawler|QuerySeekerSpider|ScollSpider|Trends Crawler|USyd-NLP-Spider|SiteCat Webbot|BotName\/\$BotVersion|123metaspider-Bot|1470\.net crawler|50\.nu|8bo Crawler Bot|Aboundex|Accoona-[A-z]{1,30}-Agent|AdsBot-Google(?:-[a-z]{1,30}|)|altavista|AppEngine-Google|archive.{0,30}\.org_bot|archiver|Ask Jeeves|[Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]{1,30})(?:-[A-Za-z]{1,30}|)|bingbot|BingPreview|blitzbot|BlogBridge|Bloglovin|BoardReader Blog Indexer|BoardReader Favicon Fetcher|boitho.com-dc|BotSeer|BUbiNG|\b\w{0,30}favicon\w{0,30}\b|\bYeti(?:-[a-z]{1,30}|)|Catchpoint(?: bot|)|[Cc]harlotte|Checklinks|clumboot|Comodo HTTP\(S\) Crawler|Comodo-Webinspector-Crawler|ConveraCrawler|CRAWL-E|CrawlConvera|Daumoa(?:-feedfetcher|)|Feed Seeker Bot|Feedbin|findlinks|Flamingo_SearchEngine|FollowSite Bot|furlbot|Genieo|gigabot|GomezAgent|gonzo1|(?:[a-zA-Z]{1,30}-|)Googlebot(?:-[a-zA-Z]{1,30}|)|Google SketchUp|grub-client|gsa-crawler|heritrix|HiddenMarket|holmes|HooWWWer|htdig|ia_archiver|ICC-Crawler|Icarus6j|ichiro(?:/mobile|)|IconSurf|IlTrovatore(?:-Setaccio|)|InfuzApp|Innovazion Crawler|InternetArchive|IP2[a-z]{1,30}Bot|jbot\b|KaloogaBot|Kraken|Kurzor|larbin|LEIA|LesnikBot|Linguee Bot|LinkAider|LinkedInBot|Lite Bot|Llaut|lycos|Mail\.RU_Bot|masscan|masidani_bot|Mediapartners-Google|Microsoft .{0,30} Bot|mogimogi|mozDex|MJ12bot|msnbot(?:-media {0,2}|)|msrbot|Mtps Feed Aggregation System|netresearch|Netvibes|NewsGator[^/]{0,30}|^NING|Nutch[^/]{0,30}|Nymesis|ObjectsSearch|OgScrper|Orbiter|OOZBOT|PagePeeker|PagesInventory|PaxleFramework|Peeplo Screenshot Bot|PHPCrawl|PlantyNet_WebRobot|Pompos|Qwantify|Read%20Later|Reaper|RedCarpet|Retreiver|Riddler|Rival IQ|scooter|Scrapy|Scrubby|searchsight|seekbot|semanticdiscovery|SemrushBot|Simpy|SimplePie|SEOstats|SimpleRSS|SiteCon|Slackbot-LinkExpanding|Slack-ImgProxy|Slurp|snappy|Speedy Spider|Squrl Java|Stringer|TheUsefulbot|ThumbShotsBot|Thumbshots\.ru|Tiny Tiny RSS|Twitterbot|WhatsApp|URL2PNG|Vagabondo|VoilaBot|^vortex|Votay bot|^voyager|WASALive.Bot|Web-sniffer|WebThumb|WeSEE:[A-z]{1,30}|WhatWeb|WIRE|WordPress|Wotbox|www\.almaden\.ibm\.com|Xenu(?:.s|) Link Sleuth|Xerka [A-z]{1,30}Bot|yacy(?:bot|)|YahooSeeker|Yahoo! Slurp|Yandex\w{1,30}|YodaoBot(?:-[A-z]{1,30}|)|YottaaMonitor|Yowedo|^Zao|^Zao-Crawler|ZeBot_www\.ze\.bz|ZooShot|ZyBorg|ArcGIS Hub Indexer)(?:[ /]v?(\d+)(?:\.(\d+)(?:\.(\d+)|)|)|)'
+  - regex: '(CSimpleSpider|Cityreview Robot|CrawlDaddy|CrawlFire|Finderbots|Index crawler|Job Roboter|KiwiStatus Spider|Lijit Crawler|QuerySeekerSpider|ScollSpider|Trends Crawler|USyd-NLP-Spider|SiteCat Webbot|BotName\/\$BotVersion|123metaspider-Bot|1470\.net crawler|50\.nu|8bo Crawler Bot|Aboundex|Accoona-[A-z]{1,30}-Agent|AdsBot-Google(?:-[a-z]{1,30}|)|altavista|AppEngine-Google|archive.{0,30}\.org_bot|archiver|Ask Jeeves|[Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]{1,30})(?:-[A-Za-z]{1,30}|)|bingbot|BingPreview|blitzbot|BlogBridge|Bloglovin|BoardReader Blog Indexer|BoardReader Favicon Fetcher|boitho.com-dc|BotSeer|BUbiNG|\b\w{0,30}favicon\w{0,30}\b|\bYeti(?:-[a-z]{1,30}|)|Catchpoint(?: bot|)|[Cc]harlotte|Checklinks|clumboot|Comodo HTTP\(S\) Crawler|Comodo-Webinspector-Crawler|ConveraCrawler|CRAWL-E|CrawlConvera|Daumoa(?:-feedfetcher|)|Feed Seeker Bot|Feedbin|findlinks|Flamingo_SearchEngine|FollowSite Bot|furlbot|Genieo|gigabot|GomezAgent|gonzo1|(?:[a-zA-Z]{1,30}-|)Googlebot(?:-[a-zA-Z]{1,30}|)|GoogleOther|Google SketchUp|grub-client|gsa-crawler|heritrix|HiddenMarket|holmes|HooWWWer|htdig|ia_archiver|ICC-Crawler|Icarus6j|ichiro(?:/mobile|)|IconSurf|IlTrovatore(?:-Setaccio|)|InfuzApp|Innovazion Crawler|InternetArchive|IP2[a-z]{1,30}Bot|jbot\b|KaloogaBot|Kraken|Kurzor|larbin|LEIA|LesnikBot|Linguee Bot|LinkAider|LinkedInBot|Lite Bot|Llaut|lycos|Mail\.RU_Bot|masscan|masidani_bot|Mediapartners-Google|Microsoft .{0,30} Bot|mogimogi|mozDex|MJ12bot|msnbot(?:-media {0,2}|)|msrbot|Mtps Feed Aggregation System|netresearch|Netvibes|NewsGator[^/]{0,30}|^NING|Nutch[^/]{0,30}|Nymesis|ObjectsSearch|OgScrper|Orbiter|OOZBOT|PagePeeker|PagesInventory|PaxleFramework|Peeplo Screenshot Bot|PHPCrawl|PlantyNet_WebRobot|Pompos|Qwantify|Read%20Later|Reaper|RedCarpet|Retreiver|Riddler|Rival IQ|scooter|Scrapy|Scrubby|searchsight|seekbot|semanticdiscovery|SemrushBot|Simpy|SimplePie|SEOstats|SimpleRSS|SiteCon|Slackbot-LinkExpanding|Slack-ImgProxy|Slurp|snappy|Speedy Spider|Squrl Java|Stringer|TheUsefulbot|ThumbShotsBot|Thumbshots\.ru|Tiny Tiny RSS|Twitterbot|WhatsApp|URL2PNG|Vagabondo|VoilaBot|^vortex|Votay bot|^voyager|WASALive.Bot|Web-sniffer|WebThumb|WeSEE:[A-z]{1,30}|WhatWeb|WIRE|WordPress|Wotbox|www\.almaden\.ibm\.com|Xenu(?:.s|) Link Sleuth|Xerka [A-z]{1,30}Bot|yacy(?:bot|)|YahooSeeker|Yahoo! Slurp|Yandex\w{1,30}|YodaoBot(?:-[A-z]{1,30}|)|YottaaMonitor|Yowedo|^Zao|^Zao-Crawler|ZeBot_www\.ze\.bz|ZooShot|ZyBorg|ArcGIS Hub Indexer|GPTBot)(?:[ /]v?(\d+)(?:\.(\d+)(?:\.(\d+)|)|)|)'
 
   # AWS S3 Clients
   # must come before "Bots General matcher" to catch "boto"/"boto3" before "bot"
-  - regex: '\b(Boto3?|JetS3t|aws-(?:cli|sdk-(?:cpp|go|java|nodejs|ruby2?|dotnet-(?:\d{1,2}|core)))|s3fs)/(\d+)\.(\d+)(?:\.(\d+)|)'
+  - regex: '\b(Boto3?|JetS3t|aws-(?:cli|sdk-(?:cpp|go|go-v\d|java|nodejs|ruby2?|dotnet-(?:\d{1,2}|core)))|s3fs)/(\d+)\.(\d+)(?:\.(\d+)|)'
 
   # SAFE FME
   - regex: '(FME)\/(\d+\.\d+)\.(\d+)\.(\d+)'
@@ -179,6 +179,9 @@ user_agent_parsers:
   - regex: '\[FB.{0,300};'
     family_replacement: 'Facebook'
 
+  # RecipeRadar crawler
+  - regex: '(RecipeRadar)/(\d+)\.(\d+)(?:\.(\d+)|)'
+
   # Bots General matcher 'name/0.0'
   - regex: '^.{0,200}?(?:\/[A-Za-z0-9\.]{0,50}|) {0,2}([A-Za-z0-9 \-_\!\[\]:]{0,50}(?:[Aa]rchiver|[Ii]ndexer|[Ss]craper|[Bb]ot|[Ss]pider|[Cc]rawl[a-z]{0,50}))[/ ](\d+)(?:\.(\d+)(?:\.(\d+)|)|)'
   # Bots containing bot(but not CUBOT)
@@ -215,6 +218,16 @@ user_agent_parsers:
   # Twitter
   - regex: '(Twitter for (?:iPhone|iPad)|TwitterAndroid)(?:\/(\d+)\.(\d+)|)'
     family_replacement: 'Twitter'
+  # TikTok
+  - regex: '(musical_ly) app_version\/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'TikTok'
+  - regex: '(musical_ly_)(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'TikTok'
+  - regex: '(BytedanceWebview)\/[a-z0-9]+'
+    family_replacement: 'TikTok'
+  # KakaoTalk
+  - regex: 'Mozilla.{1,200}Mobile.{1,100}(KAKAOTALK)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'KakaoTalk'
 
   # Phantom app
   - regex: 'Mozilla.{1,200}Mobile.{1,100}(Phantom\/ios|Phantom\/android).(\d+)\.(\d+)\.(\d+)'
@@ -394,7 +407,7 @@ user_agent_parsers:
   - regex: '(Instabridge)/(\d+)(?:\.(\d+)|)(?:\.(\d+)|)'
 
   # Aloha Browser
-  - regex: '(AlohaBrowser)/(\d+)\.(\d+)\.(\d+)(?:\.(\d+)|)'
+  - regex: '(AlohaBrowser|ABB)/(\d+)\.(\d+)\.(\d+)(?:\.(\d+)|)'
     family_replacement: 'Aloha Browser'
 
   # Brave Browser https://brave.com/ , should go before Safari and Chrome Mobile
@@ -415,7 +428,7 @@ user_agent_parsers:
     family_replacement: 'Edge Mobile'
 
   # Oculus Browser, should go before Samsung Internet
-  - regex: '(OculusBrowser)/(\d+)\.(\d+).0.0(?:\.([0-9\-]+)|)'
+  - regex: '(OculusBrowser)/(\d+)\.(\d+)(?:\.([0-9\-]+)|)'
     family_replacement: 'Oculus Browser'
 
   # Samsung Internet (based on Chrome, but lacking some features)
@@ -492,6 +505,12 @@ user_agent_parsers:
   - regex: '(Ecosia) android@(\d+)(?:\.(\d+)|)(?:\.(\d+)|)(?:\.(\d+)|)'
     family_replacement: 'Ecosia Android'
 
+  # VivoBrowser
+  - regex: '(VivoBrowser)\/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+
+  # HiBrowser
+  - regex: '(HiBrowser)\/v(\d+)\.(\d+)\.(\d+)\.(\d+)'
+
   # Chrome Mobile
   - regex: 'Version/.{1,300}(Chrome)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
     family_replacement: 'Chrome Mobile WebView'
@@ -569,6 +588,85 @@ user_agent_parsers:
   - regex: '^(surveyon)/(\d+)\.(\d+)\.(\d+)'
     family_replacement: 'Surveyon'
 
+  # 115 Browser
+  - regex: '(115Browser)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: '115 Browser'
+
+ # Avira
+  - regex: '(Avira)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Avira'
+
+  # CCleaner Browser
+  - regex: '(CCleaner)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'CCleaner'
+
+  # Norton
+  - regex: '(Norton)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Norton'
+
+  # Quark
+  - regex: '(Quark)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Quark'
+  # Quark PC
+  - regex: '(QuarkPC)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Quark PC'
+
+  # Smart Lenovo Browser
+  - regex: '(SLBrowser)/(\d+)\.(\d+)\.(\d+)\.(\d+) SLBChan/(\d+)'
+    family_replacement: 'Smart Lenovo Browser'
+  
+  # Atom Browser
+  - regex: '(Atom)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Atom Browser'
+
+  # 360 Secure Browser
+  - regex: '(Chrome)/\d+\.\d+\.\d+\.\d+ .* QIHU 360(?:SEi18n|ENT)'
+    family_replacement: '360 Secure Browser'
+
+  # Decentr Web3 Browser
+  - regex: '(Decentr)'
+    family_replacement: 'Decentr Web3 Browser'
+
+  # Sparrow Browser
+  - regex: '(Sparrow)'
+    family_replacement: 'Sparrow Browser'
+
+  # Chromium GOST Browser
+  - regex: '(Chromium GOST)'
+    family_replacement: 'Chromium GOST Browser'
+
+  # AOL Shield Browser
+  - regex: '(AOLShield)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'AOL Shield Browser'
+
+  # Hola Browser
+  - regex: '(Hola)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Hola Browser'
+
+  # Craving Explorer Browser
+  - regex: '(CravingExplorer)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Craving Explorer Browser'
+
+  # Talon Cyber Security Browser
+  - regex: '(Talon)'
+    family_replacement: 'Talon Cyber Security Browser'
+
+  # QAX Browser
+  - regex: '(Qaxbrowser)'
+    family_replacement: 'QAX Browser'
+
+  # AOL Desktop Gold Browser
+  - regex: '(ADG)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'AOL Desktop Gold Browser'
+
+  # Sber Browser
+  - regex: '(SberBrowser)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Sber Browser'
+
+  # JiSu Browser
+  - regex: '(JiSu)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'JiSu Browser'
+
   #### END SPECIAL CASES TOP ####
 
   #### MAIN CASES - this catches > 50% of all browsers ####
@@ -1387,6 +1485,13 @@ os_parsers:
   # Box Drive and Box Sync on Mac OS X use OSX version numbers, not Darwin
   - regex: '^Box.{0,200};(Darwin)/(10)\.(1\d)(?:\.(\d+)|)'
     os_replacement: 'Mac OS X'
+  
+  ##########
+  # Hashicorp API
+  # APN/1.0 HashiCorp/1.0 Terraform/1.8.0 (+https://www.terraform.io) terraform-provider-aws/4.67.0 (+https://registry.terraform.io/providers/hashicorp/aws) aws-sdk-go/1.44.261 (go1.19.8; darwin; arm64)
+  ##########
+  - regex: 'darwin; arm64'
+    os_replacement: 'Mac OS X'
 
   ##########
   # iOS
@@ -1672,29 +1777,27 @@ os_parsers:
   - regex: 'CFNetwork/.{0,100} Darwin/(21)\.\d+'
     os_replacement: 'iOS'
     os_v1_replacement: '15'
-  - regex: 'CFNetwork/.{0,100} Darwin/22\.0\.\d+'
+  - regex: 'CFNetwork/.{0,100} Darwin/22\.([0-5])\.\d+'
     os_replacement: 'iOS'
     os_v1_replacement: '16'
-    os_v2_replacement: '0'
-  - regex: 'CFNetwork/.{0,100} Darwin/22\.1\.\d+'
-    os_replacement: 'iOS'
-    os_v1_replacement: '16'
-    os_v2_replacement: '1'
-  - regex: 'CFNetwork/.{0,100} Darwin/22\.2\.\d+'
-    os_replacement: 'iOS'
-    os_v1_replacement: '16'
-    os_v2_replacement: '2'
-  - regex: 'CFNetwork/.{0,100} Darwin/22\.3\.\d+'
-    os_replacement: 'iOS'
-    os_v1_replacement: '16'
-    os_v2_replacement: '3'
-  - regex: 'CFNetwork/.{0,100} Darwin/22\.4\.\d+'
-    os_replacement: 'iOS'
-    os_v1_replacement: '16'
-    os_v2_replacement: '4'
+    os_v2_replacement: '$1'
   - regex: 'CFNetwork/.{0,100} Darwin/(22)\.\d+'
     os_replacement: 'iOS'
     os_v1_replacement: '16'
+  - regex: 'CFNetwork/.{0,100} Darwin/23\.([0-5])\.\d+'
+    os_replacement: 'iOS'
+    os_v1_replacement: '17'
+    os_v2_replacement: '$1'
+  - regex: 'CFNetwork/.{0,100} Darwin/(23)\.\d+'
+    os_replacement: 'iOS'
+    os_v1_replacement: '17'
+  - regex: 'CFNetwork/.{0,100} Darwin/24\.([0-5])\.\d+'
+    os_replacement: 'iOS'
+    os_v1_replacement: '18'
+    os_v2_replacement: '$1'
+  - regex: 'CFNetwork/.{0,100} Darwin/(24)\.\d+'
+    os_replacement: 'iOS'
+    os_v1_replacement: '18'
   - regex: 'CFNetwork/.{0,100} Darwin/'
     os_replacement: 'iOS'
 
@@ -1889,6 +1992,21 @@ os_parsers:
   # Roku Digital-Video-Players https://www.roku.com/
   - regex: '^(Roku)/DVP-(\d+)\.(\d+)'
 
+  ##########
+  # Amazon S3 client boto3
+  # Hasicorp API
+  # Boto3/1.28.62 md/Botocore#1.31.62 ua/2.0 os/macos#22.4.0 md/arch#arm64 lang/python#3.11.6 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.62
+  # APN/1.0 HashiCorp/1.0 Terraform/1.8.1 (+https://www.terraform.io) terraform-provider-aws/4.67.0 (+https://registry.terraform.io/providers/hashicorp/aws) aws-sdk-go-v2/1.18.0 os/macos lang/go/1.19.8 md/GOOS/darwin md/GOARCH/arm64 api/identitystore/1.16.11
+  ##########
+  - regex: 'os\/macos[#]?(\d*)[.]?(\d*)[.]?(\d*)'
+    os_replacement: 'Mac OS X'
+    os_v1_replacement: '$1'
+    os_v2_replacement: '$2'
+    os_v3_replacement: '$3'
+
+  # Huawei HarmonyOS
+  - regex: '(HarmonyOS)[\s;]+(\d+|)\.?(\d+|)\.?(\d+|)'
+
 device_parsers:
 
   #########
@@ -5905,7 +6023,7 @@ device_parsers:
   ##########
   # Spiders (this is a hack...)
   ##########
-  - regex: '^.{0,100}(bot|BUbiNG|zao|borg|DBot|oegp|silk|Xenu|zeal|^NING|CCBot|crawl|htdig|lycos|slurp|teoma|voila|yahoo|Sogou|CiBra|Nutch|^Java/|^JNLP/|Daumoa|Daum|Genieo|ichiro|larbin|pompos|Scrapy|snappy|speedy|spider|msnbot|msrbot|vortex|^vortex|crawler|favicon|indexer|Riddler|scooter|scraper|scrubby|WhatWeb|WinHTTP|bingbot|BingPreview|openbot|gigabot|furlbot|polybot|seekbot|^voyager|archiver|Icarus6j|mogimogi|Netvibes|blitzbot|altavista|charlotte|findlinks|Retreiver|TLSProber|WordPress|SeznamBot|ProoXiBot|wsr\-agent|Squrl Java|EtaoSpider|PaperLiBot|SputnikBot|A6\-Indexer|netresearch|searchsight|baiduspider|YisouSpider|ICC\-Crawler|http%20client|Python-urllib|dataparksearch|converacrawler|Screaming Frog|AppEngine-Google|YahooCacheSystem|fast\-webcrawler|Sogou Pic Spider|semanticdiscovery|Innovazion Crawler|facebookexternalhit|Google.{0,200}/\+/web/snippet|Google-HTTP-Java-Client|BlogBridge|IlTrovatore-Setaccio|InternetArchive|GomezAgent|WebThumbnail|heritrix|NewsGator|PagePeeker|Reaper|ZooShot|holmes|NL-Crawler|Pingdom|StatusCake|WhatsApp|masscan|Google Web Preview|Qwantify|Yeti|OgScrper)'
+  - regex: '^.{0,100}(bot|BUbiNG|zao|borg|DBot|oegp|silk|Xenu|zeal|^NING|CCBot|crawl|htdig|lycos|slurp|teoma|voila|yahoo|Sogou|CiBra|Nutch|^Java/|^JNLP/|Daumoa|Daum|Genieo|ichiro|larbin|pompos|Scrapy|snappy|speedy|spider|msnbot|msrbot|vortex|^vortex|crawler|favicon|indexer|Riddler|scooter|scraper|scrubby|WhatWeb|WinHTTP|bingbot|BingPreview|openbot|gigabot|furlbot|polybot|seekbot|^voyager|archiver|Icarus6j|mogimogi|Netvibes|blitzbot|altavista|charlotte|findlinks|Retreiver|TLSProber|WordPress|SeznamBot|ProoXiBot|wsr\-agent|Squrl Java|EtaoSpider|PaperLiBot|SputnikBot|A6\-Indexer|netresearch|searchsight|baiduspider|YisouSpider|ICC\-Crawler|http%20client|Python-urllib|dataparksearch|converacrawler|Screaming Frog|AppEngine-Google|YahooCacheSystem|fast\-webcrawler|Sogou Pic Spider|semanticdiscovery|Innovazion Crawler|facebookexternalhit|Google.{0,200}/\+/web/snippet|Google-HTTP-Java-Client|BlogBridge|IlTrovatore-Setaccio|InternetArchive|GomezAgent|WebThumbnail|heritrix|NewsGator|PagePeeker|Reaper|ZooShot|holmes|NL-Crawler|Pingdom|StatusCake|WhatsApp|masscan|Google Web Preview|Qwantify|Yeti|OgScrper|RecipeRadar|GPTBot)'
     regex_flag: 'i'
     device_replacement: 'Spider'
     brand_replacement: 'Spider'

db/migrations/20240711224103_create_clicks.sql

@@ -4,7 +4,6 @@ CREATE TABLE clicks (
 	id TEXT PRIMARY KEY NOT NULL,
 	link_id TEXT NOT NULL,
 	user_agent TEXT,
-	language TEXT,
 	browser TEXT,
 	os TEXT,
 	source TEXT,

db/migrations/20250316102350_add_country_to_clicks.sql

@@ -0,0 +1,8 @@
+-- +micrate Up
+-- SQL in section 'Up' is executed when this migration is applied
+ALTER TABLE clicks ADD COLUMN country TEXT;
+ALTER TABLE clicks RENAME COLUMN source TO referer;
+
+-- +micrate Down
+-- SQL section 'Down' is executed when this migration is rolled back
+ALTER TABLE clicks RENAME COLUMN referer TO source;

db/migrations/20250316111734_replace_unkwown_with_null.sql

@@ -0,0 +1,13 @@
+-- +micrate Up
+-- SQL in section 'Up' is executed when this migration is applied
+UPDATE clicks SET user_agent = NULL WHERE user_agent = 'Unknown';
+UPDATE clicks SET browser = NULL WHERE browser = 'Unknown';
+UPDATE clicks SET os = NULL WHERE os = 'Unknown';
+UPDATE clicks SET referer = NULL WHERE referer = 'Unknown';
+
+-- +micrate Down
+-- SQL section 'Down' is executed when this migration is rolled back
+UPDATE clicks SET user_agent = 'Unknown' WHERE user_agent IS NULL;
+UPDATE clicks SET browser = 'Unknown' WHERE browser IS NULL;
+UPDATE clicks SET os = 'Unknown' WHERE os IS NULL;
+UPDATE clicks SET referer = 'Unknown' WHERE referer IS NULL;

docs/API.md

@@ -0,0 +1,125 @@
+# API Reference
+
+1. **Ping the API**
+
+   - Endpoint: `GET /api/ping`
+   - Payload: None
+   - Response Example
+     ```json
+     {
+       "data": "pong"
+     }
+     ```
+
+2. **Redirect by Slug**
+
+   - Endpoint: `GET /:slug`
+   - Payload: None
+   - Response: 301
+
+3. **List All Links**
+
+   - Endpoint: `GET /api/links`
+   - Headers: `X-Api-Key`
+   - Payload: None
+   - Response Example
+     ```json
+     {
+       "data": [
+         {
+           "id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db",
+           "refer": "http://localhost:4000/3wP4BQ",
+           "origin": "https://monocuco.donado.co",
+           "clicks": [
+             {
+               "id": "730e2202-58f9-478c-a24c-f1c561df6716",
+               "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0",
+               "country": "DE",
+               "browser": "Firefox",
+               "os": "Mac OS X",
+               "referer": "Direct",
+               "created_at": "2024-07-12T19:25:22Z"
+             }
+           ]
+         }
+       ]
+     }
+     ```
+
+4. **List link by ID**
+
+   - Endpoint: `GET /api/links/:id`
+   - Headers: `X-Api-Key`
+   - Payload: None
+   - Response Example
+     ```json
+     {
+       "data": {
+         "id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db",
+         "refer": "http://localhost:4000/3wP4BQ",
+         "origin": "https://monocuco.donado.co",
+         "clicks": [
+           {
+             "id": "730e2202-58f9-478c-a24c-f1c561df6716",
+             "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0",
+             "country": "DE",
+             "browser": "Firefox",
+             "os": "Mac OS X",
+             "referer": "Direct",
+             "created_at": "2024-07-12T19:25:22Z"
+           }
+         ]
+       }
+     }
+     ```
+
+5. **Create new link**
+
+   - Endpoint\*\*: `POST /api/links`
+   - Payload:
+     ```json
+     {
+       "url": "https://example.com"
+     }
+     ```
+   - Headers: `X-Api-Key`
+   - Response Example:
+     ```json
+     {
+       "data": {
+         "id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db",
+         "refer": "http://localhost:4000/3wP4BQ",
+         "origin": "https://example.com",
+         "clicks": []
+       }
+     }
+     ```
+
+6. **Update an existing link by ID**
+
+   - Endpoint: `PUT /api/links/:id`
+   - Payload:
+     ```json
+     {
+       "url": "https://newexample.com"
+     }
+     ```
+   - Headers: `X-Api-Key`
+   - Response Example:
+     ```json
+     {
+       "data": {
+         "id": "84f0c7a4-8c4e-4665-b676-cb9c5e40f1db",
+         "refer": "http://localhost:4000/3wP4BQ",
+         "origin": "https://newexample.com",
+         "clicks": []
+       }
+     }
+     ```
+
+7. **Delete a link by ID**
+
+  - Endpoint: `DELETE /api/links/:id`
+  - Payload: None
+  - Headers: `X-Api-Key`
+  - Response: 204

docs/SETUP.md

@@ -0,0 +1,154 @@
+## CLI
+
+```
+Usage: ./cli [options]
+Options:
+  --create-user=NAME    Create a new user with the given name
+  --list-users          List all users
+  --delete-user=USER_ID Delete a user by ID
+```
+
+## Run It Anywhere
+
+### Docker Compose
+
+```bash
+docker-compose up
+
+# Optional: Generate an api key
+# docker-compose exec -it app cli --create-user=Admin
+```
+
+### Docker CLI
+
+```bash
+docker run \
+    --name bit \
+    -p 4000:4000 \
+    -e ENV="production" \
+    -e DATABASE_URL="sqlite3://./sqlite/data.db?journal_mode=wal&synchronous=normal&foreign_keys=true" \
+    -e APP_URL="http://localhost:4000" \
+    -e ADMIN_NAME="Admin" \
+    -e ADMIN_API_KEY=$(openssl rand -base64 32) \
+    sjdonado/bit
+
+# Optional: Generate an api key
+# docker exec -it bit cli --create-user=Admin
+```
+
+### Self-Hosted with Dokku
+
+```dockerfile
+FROM sjdonado/bit
+```
+
+```bash
+dokku apps:create bit
+
+dokku domains:set bit bit.donado.co
+dokku letsencrypt:enable bit
+
+dokku storage:ensure-directory bit-sqlite
+dokku storage:mount bit /var/lib/dokku/data/storage/bit-sqlite:/usr/src/app/sqlite/
+
+dokku config:set bit DATABASE_URL="sqlite3://./sqlite/data.db?journal_mode=wal&synchronous=normal&foreign_keys=true" APP_URL=https://bit.donado.co ADMIN_NAME=Admin ADMIN_API_KEY=$(openssl rand -base64 32)
+
+dokku ports:add bit http:80:4000
+dokku ports:add bit https:443:4000
+
+# Optional: Generate an api key
+# dokku run bit cli --create-user=Admin
+```
+
+## Local Development
+
+### Requirements
+- Crystal 1.12+
+- Shards package manager
+- SQLite3
+
+### Install Dependencies
+- linux
+```bash
+sudo apt-get update && sudo apt-get install -y crystal libssl-dev libsqlite3-dev
+```
+
+- macos
+```bash
+brew tap amberframework/micrate
+brew install micrate
+```
+
+### Install Shards and Run
+
+```bash
+shards run bit
+```
+
+- Generate the `X-Api-Key`
+
+```bash
+shards run cli -- --create-user=Admin
+```
+
+- Run tests
+
+```bash
+ENV=test crystal spec
+```
+
+## Benchmark
+
+CPU: Apple M3 Pro
+
+```
+> colima start --cpu 1 --memory 1
+INFO[0000] starting colima
+INFO[0000] runtime: docker
+INFO[0001] starting ...                                  context=vm
+INFO[0076] provisioning ...                              context=docker
+INFO[0077] starting ...                                  context=docker
+INFO[0077] done
+
+> ./benchmark.sh
+Setting up...
+[+] Running 2/2
+ ✔ Network bit_default  Created                                                                                                                                                                     0.0s
+ ✔ Container bit        Started                                                                                                                                                                     0.1s
+Captured API Key: v-8gljT0WjMhQECito3e5g
+Waiting for the application to be ready...
+HTTP/1.1 200 OK
+Connection: keep-alive
+Content-Type: application/json
+Date: Sun, 16 Mar 2025 10:51:22 GMT
+Access-Control-Allow-Origin: *
+Access-Control-Allow-Methods: GET, POST, PUT, DELETE, OPTIONS
+Access-Control-Allow-Headers: Content-Type, Accept, Origin, X-Api-Key
+Content-Length: 13
+
+Starting resource usage monitoring...
+Creating 10000 short links with 100 conrurrent requests...
+Link creation complete: 10000 links created.
+Fetching all created links from /api/links...
+Selected link for benchmarking: http://localhost:4000/pKtTjA
+Starting benchmark with Bombardier...
+Bombarding http://localhost:4000/pKtTjA with 10000 request(s) using 100 connection(s)
+ 10000 / 10000 [======================================================================================================================================================================] 100.00% 1424/s 7s
+Done!
+Statistics        Avg      Stdev        Max
+  Reqs/sec      1885.24    7686.34  140641.16
+  Latency       68.00ms     6.43ms    89.56ms
+  HTTP codes:
+    1xx - 0, 2xx - 0, 3xx - 10000, 4xx - 0, 5xx - 0
+    others - 0
+  Throughput:   625.54KB/s
+Benchmark completed.
+Analyzing resource usage...
+**** Results ****
+Average CPU Usage: 42.98%
+Average Memory Usage: 33.25 MiB
+./benchmark.sh: line 135:  1500 Terminated: 15          monitor_resource_usage
+[+] Running 2/2
+ ✔ Container bit        Removed                                                                                                                                                                    10.1s
+ ✔ Network bit_default  Removed                                                                                                                                                                     0.0s
+```

scripts/cli.cr

@@ -19,11 +19,20 @@ OptionParser.parse do |parser|
     exit
   end
 
+  parser.on("--update-data", "Download all required data files (UA Parser and GeoLite2)") do
+    puts "=== Starting data files update ==="
+    App::Services::Cli.update_uap_regexes
+    App::Services::Cli.download_geolite_db
+    puts "=== All data files updated successfully ==="
+    exit
+  end
+
   if ARGV.empty?
     puts "Usage: ./cli [options]"
     puts "Options:"
-    puts "  --create-user=NAME  Create a new user with the given name"
-    puts "  --list-users        List all users"
-    puts "  --delete-user=USER_ID Delete a user by ID"
+    puts "  --create-user=NAME     Create a new user with the given name"
+    puts "  --list-users           List all users"
+    puts "  --delete-user=USER_ID  Delete a user by ID"
+    puts "  --update-data          Download all required data files"
   end
 end

shard.lock

@@ -20,10 +20,18 @@ shards:
     git: https://github.com/crystal-loot/exception_page.git
     version: 0.4.1
 
+  ipaddress:
+    git: https://github.com/sija/ipaddress.cr.git
+    version: 0.2.3
+
   kemal:
     git: https://github.com/kemalcr/kemal.git
     version: 1.5.0
 
+  maxminddb:
+    git: https://github.com/delef/maxminddb.cr.git
+    version: 1.5.0
+
   micrate:
     git: https://github.com/amberframework/micrate.git
     version: 0.15.1

shard.yml

@@ -1,5 +1,5 @@
 name: bit
-version: 1.4.0
+version: 1.5.0
 
 authors:
   - Juan Rodriguez <sjdonado@icloud.com>
@@ -22,7 +22,8 @@ dependencies:
     version: 0.15.1
   user_agent_parser:
     github: busyloop/user_agent_parser
-    version: 2.0.1
+  maxminddb:
+    github: delef/maxminddb.cr
 
 development_dependencies:
   dotenv:

spec/integration/link_spec.cr

@@ -82,33 +82,71 @@ describe "App::Controllers::Link" do
   end
 
   describe "Index" do
-    it "should redirect to origin domain" do
+    it "should redirect to origin domain with forwarded headers" do
       link = "https://test.com"
       test_user = create_test_user()
 
       test_link = create_test_link(test_user, link)
       serialized_link = App::Serializers::Link.new(test_link)
 
-      get(serialized_link.refer, headers: HTTP::Headers{"X-Api-Key" => test_user.api_key.to_s})
+      user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0"
+
+      get(serialized_link.refer, headers: HTTP::Headers{
+        "X-Api-Key" => test_user.api_key.to_s,
+        "User-Agent" => user_agent
+      })
 
       response.headers["Location"].should eq(link)
+      response.headers["X-Forwarded-User-Agent"].should eq(user_agent)
+      response.headers.has_key?("X-Forwarded-For").should be_true
     end
 
-    it "should create a new click after redirect" do
+    it "should create a new click after redirect with proper information" do
       link = "https://sjdonado.com"
       test_user = create_test_user()
 
       test_link = create_test_link(test_user, link)
       serialized_link = App::Serializers::Link.new(test_link)
 
-      get(serialized_link.refer, headers: HTTP::Headers{"User-Agent" => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0"})
+      user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0"
+      referer = "https://example.com/page"
+
+      get(serialized_link.refer, headers: HTTP::Headers{
+        "User-Agent" => user_agent,
+        "Referer" => referer
+      })
 
       Fiber.yield # replace yield with sleep 5 to debug errors
 
       response.headers["Location"].should eq(link)
 
+      # Verify that the click was recorded
       updated_test_link = get_test_link(test_link.id)
       updated_test_link.clicks.size.should eq(test_link.clicks.size + 1)
+
+      # Verify click details
+      latest_click = updated_test_link.clicks.last
+      latest_click.user_agent.should eq(user_agent)
+      latest_click.browser.should eq("Firefox")
+      latest_click.os.should eq("Mac OS X")
+      latest_click.referer.should eq("example.com") # Should extract host from the referer
+    end
+
+    it "should create a click with utm_source when no referer is provided" do
+      link = "https://sjdonado.com"
+      test_user = create_test_user()
+
+      test_link = create_test_link(test_user, link)
+      serialized_link = App::Serializers::Link.new(test_link)
+
+      # Add utm_source parameter
+      get("#{serialized_link.refer}?utm_source=email_campaign")
+
+      Fiber.yield
+
+      updated_test_link = get_test_link(test_link.id)
+      latest_click = updated_test_link.clicks.last
+      latest_click.referer.should eq("email_campaign")
     end
 
     it "should return 404 - link does not exist" do