chore: bump to 0.16.0

Return support for F+/F- detection via fuzzing

.actor/Dockerfile

@@ -0,0 +1,19 @@
+FROM sherlock/sherlock as sherlock
+
+# Install Node.js
+RUN apt-get update; apt-get install curl gpg -y
+RUN mkdir -p /etc/apt/keyrings
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
+RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
+RUN apt-get update && apt-get install -y curl bash git jq jo xz-utils nodejs
+
+# Install Apify CLI (node.js) for the Actor Runtime
+RUN npm -g install apify-cli
+
+# Install Dependencies for the Actor Shell Script
+RUN apt-get update && apt-get install -y bash jq jo xz-utils nodejs
+
+# Copy Actor dir with the actorization shell script
+COPY .actor/ .actor
+
+ENTRYPOINT [".actor/actor.sh"]

.actor/README.md

@@ -0,0 +1,93 @@
+# Sherlock Actor on Apify
+
+[![Sherlock Actor](https://apify.com/actor-badge?actor=netmilk/sherlock)](https://apify.com/netmilk/sherlock?fpr=sherlock)
+
+This Actor wraps the [Sherlock Project](https://sherlockproject.xyz/) to provide serverless username reconnaissance across social networks in the cloud. It helps you find usernames across multiple social media platforms without installing and running the tool locally.
+
+## What are Actors?
+[Actors](https://docs.apify.com/platform/actors?fpr=sherlock) are serverless microservices running on the [Apify Platform](https://apify.com/?fpr=sherlock). They are based on the [Actor SDK](https://docs.apify.com/sdk/js?fpr=sherlock) and can be found in the [Apify Store](https://apify.com/store?fpr=sherlock). Learn more about Actors in the [Apify Whitepaper](https://whitepaper.actor?fpr=sherlock).
+
+## Usage
+
+### Apify Console
+
+1. Go to the Apify Actor page
+2. Click "Run"
+3. In the input form, fill in **Username(s)** to search for
+4. The Actor will run and produce its outputs in the default datastore
+
+
+### Apify CLI
+
+```bash
+apify call YOUR_USERNAME/sherlock --input='{
+  "usernames": ["johndoe", "janedoe"]
+}'
+```
+
+### Using Apify API
+
+```bash
+curl --request POST \
+  --url "https://api.apify.com/v2/acts/YOUR_USERNAME~sherlock/run" \
+  --header 'Content-Type: application/json' \
+  --header 'Authorization: Bearer YOUR_API_TOKEN' \
+  --data '{
+  "usernames": ["johndoe", "janedoe"],
+  }
+}'
+```
+
+## Input Parameters
+
+The Actor accepts a JSON schema with the following structure:
+
+| Field | Type | Required | Default | Description |
+|-------|------|----------|---------|-------------|
+| `usernames` | array | Yes | - | List of usernames to search for |
+| `usernames[]` | string | Yes | "json" | Username to search for |
+
+
+### Example Input
+
+```json
+{
+  "usernames": ["techuser", "designuser"],
+}
+```
+
+## Output
+
+The Actor provides three types of outputs:
+
+### Dataset Record*
+
+| Field | Type | Required | Description |
+|-------|------|----------|-------------|
+| `username` | string | Yes | Username the search was conducted for |
+| `links` | arrray | Yes | Array with found links to the social media |
+| `links[]`| string | No | URL to the account
+
+### Example Dataset Item (JSON)
+
+```json
+{
+  "username": "johndoe",
+  "links": [
+    "https://github.com/johndoe" 
+  ]
+}
+```
+
+## Performance & Resources
+
+- **Memory Requirements**:
+  - Minimum: 512 MB RAM
+  - Recommended: 1 GB RAM for multiple usernames
+- **Processing Time**:
+  - Single username: ~1-2 minutes
+  - Multiple usernames: 2-5 minutes
+  - Varies based on number of sites checked and response times
+
+
+For more help, check the [Sherlock Project documentation](https://github.com/sherlock-project/sherlock) or raise an issue in the Actor's repository.

.actor/actor.json

@@ -0,0 +1,13 @@
+{
+  "actorSpecification": 1,
+  "name": "sherlock",
+  "version": "0.0",
+  "buildTag": "latest",
+  "environmentVariables": {},
+  "dockerFile": "./Dockerfile", 
+  "dockerContext": "../",
+  "input": "./input_schema.json",
+  "storages": {
+    "dataset": "./dataset_schema.json"
+  }
+}

.actor/actor.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+INPUT=`apify actor:get-input | jq -r .usernames[] | xargs echo`
+echo "INPUT: $INPUT"
+
+sherlock $INPUT
+
+for username in $INPUT; do
+  # escape the special meaning leading characters 
+  # https://github.com/jpmens/jo/blob/master/jo.md#description
+  safe_username=$(echo $username | sed 's/^@/\\@/' | sed 's/^:/\\:/' | sed 's/%/\\%/')
+  echo "pushing results for username: $username, content:"
+  cat $username.txt
+  sed '$d' $username.txt | jo -a | jo username=$safe_username links:=- | apify actor:push-data
+done

.actor/dataset_schema.json

@@ -0,0 +1,45 @@
+{
+    "actorSpecification": 1,
+    "fields":{
+      "title": "Sherlock actor input",
+      "description": "This is actor input schema",
+      "type": "object",
+      "schemaVersion": 1,
+      "properties": {
+        "links": {
+          "title": "Links to accounts",
+          "type": "array",
+          "description": "A list of social media accounts found for the uername"
+        },
+        "username": {
+          "title": "Lookup username",
+          "type": "string",
+          "description": "Username the lookup was performed for"
+        }
+      },
+      "required": [
+        "username", 
+        "links"
+      ]
+    },
+    "views": {
+        "overview": {
+            "title": "Overview",
+            "transformation": {
+              "fields": [
+                "username",
+                "links"
+              ],
+            },
+            "display": {
+               "component": "table",
+               "links": {
+                 "label": "Links"
+               },
+               "username":{
+                 "label": "Username"
+               }
+            }
+        }
+    }
+}

.actor/input_schema.json

@@ -0,0 +1,18 @@
+{
+  "title": "Sherlock actor input",
+  "description": "This is actor input schema",
+  "type": "object",
+  "schemaVersion": 1,
+  "properties": {
+    "usernames": {
+      "title": "Usernames to hunt down",
+      "type": "array",
+      "description": "A list of usernames to be checked for existence across social media",
+      "editor": "stringList",
+      "prefill": ["johndoe"]
+    }
+  },
+  "required": [
+    "usernames"
+  ]
+}

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -19,6 +19,15 @@ body:
         - Other (indicate below)
     validations:
       required: true
+  - type: input
+    id: package-version
+    attributes:
+      label: Package version
+      description: |
+        Knowing the version of the package you are using can help us diagnose your issue more quickly.
+        You can find the version by running `sherlock --version`.
+    validations:
+      required: true
   - type: textarea
     id: description
     attributes:

.github/workflows/exclusions.yml

@@ -0,0 +1,80 @@
+name: Exclusions Updater
+
+on:
+  schedule:
+    #- cron: '0 5 * * 0'  # Runs at 05:00 every Sunday
+    - cron: '0 5 * * *' # Runs at 05:00 every day
+  workflow_dispatch:
+
+jobs:
+  update-exclusions:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.13'
+
+      - name: Install Poetry
+        uses: abatilo/actions-poetry@v4
+        with:
+          poetry-version: 'latest'
+
+      - name: Install dependencies
+        run: |
+          poetry install --no-interaction --with dev
+
+      - name: Run false positive tests
+        run: |
+          $(poetry env activate)
+          pytest -q --tb no -m validate_targets_fp -n 20 | tee fp_test_results.txt
+          deactivate
+
+      - name: Parse false positive detections by desired categories
+        run: |
+          grep -oP '(?<=test_false_pos\[)[^\]]+(?=\].*result was Claimed)' fp_test_results.txt \
+            | sort -u > false_positive_exclusions.txt
+          grep -oP '(?<=test_false_pos\[)[^\]]+(?=\].*result was WAF)' fp_test_results.txt \
+            | sort -u > waf_hits.txt
+
+      - name: Detect if exclusions list changed
+        id: detect_changes
+        run: |
+          git fetch origin exclusions || true
+
+          if git show origin/exclusions:exclusions.txt >/dev/null 2>&1; then
+            # If the exclusions branch and file exist, compare
+            if git diff --quiet origin/exclusions -- exclusions.txt; then
+              echo "exclusions_changed=false" >> "$GITHUB_OUTPUT"
+            else
+              echo "exclusions_changed=true" >> "$GITHUB_OUTPUT"
+            fi
+          else
+            # If the exclusions branch or file do not exist, treat as changed
+            echo "exclusions_changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Quantify and display results
+        run: |
+          FP_COUNT=$(wc -l < false_positive_exclusions.txt | xargs)
+          WAF_COUNT=$(wc -l < waf_hits.txt | xargs)
+          echo ">>> Found $FP_COUNT false positives and $WAF_COUNT WAF hits."
+          echo ">>> False positive exclusions:" && cat false_positive_exclusions.txt
+          echo ">>> WAF hits:" && cat waf_hits.txt
+
+      - name: Commit and push exclusions list
+        if: steps.detect_changes.outputs.exclusions_changed == 'true'
+        run: |
+          git config user.name "Paul Pfeister (automation)"
+          git config user.email "code@pfeister.dev"
+
+          git fetch origin exclusions || true # Allows creation of branch if deleted
+          git checkout -B exclusions origin/exclusions || (git checkout --orphan exclusions && git rm -rf .)
+
+          git add false_positive_exclusions.txt
+
+          git commit -m "auto: update exclusions list" || echo "No changes to commit"
+          git push origin exclusions

.github/workflows/regression.yml

@@ -49,11 +49,10 @@ jobs:
           macos-latest,
         ]
         python-version: [
-          '3.8',
-          '3.9',
           '3.10',
           '3.11',
           '3.12',
+          '3.13',
         ]
     steps:
       - uses: actions/checkout@v4

Dockerfile

@@ -2,13 +2,12 @@
   # 1. Update the version tag in the Dockerfile to match the version in sherlock/__init__.py
   # 2. Update the VCS_REF tag to match the tagged version's FULL commit hash
   # 3. Build image with BOTH latest and version tags
-    # i.e. `docker build -t sherlock/sherlock:0.15.0 -t sherlock/sherlock:latest .`
+    # i.e. `docker build -t sherlock/sherlock:0.16.0 -t sherlock/sherlock:latest .`
 
 FROM python:3.12-slim-bullseye as build
 WORKDIR /sherlock
 
-RUN apt-get update \
-  pip3 install --no-cache-dir --upgrade pip
+RUN pip3 install --no-cache-dir --upgrade pip
 
 FROM python:3.12-slim-bullseye
 WORKDIR /sherlock

devel/site-list.py

@@ -4,8 +4,11 @@
 import json
 import os
 
+
+DATA_REL_URI: str = "sherlock_project/resources/data.json"
+
 # Read the data.json file
-with open("sherlock/resources/data.json", "r", encoding="utf-8") as data_file:
+with open(DATA_REL_URI, "r", encoding="utf-8") as data_file:
     data: dict = json.load(data_file)
 
 # Removes schema-specific keywords for proper processing
@@ -27,7 +30,7 @@ with open("output/sites.mdx", "w") as site_file:
         site_file.write(f"1. [{social_network}]({url_main}) {is_nsfw}\n")
 
 # Overwrite the data.json file with sorted data
-with open("sherlock/resources/data.json", "w") as data_file:
+with open(DATA_REL_URI, "w") as data_file:
     sorted_data = json.dumps(data, indent=2, sort_keys=True)
     data_file.write(sorted_data)
     data_file.write("\n")

docs/README.md

@@ -1,6 +1,6 @@
-<p align=center>
+<p align="center">
   <br>
-  <a href="https://sherlock-project.github.io/" target="_blank"><img src="images/sherlock-logo.png"/></a>
+  <a href="https://sherlock-project.github.io/" target="_blank"><img src="images/sherlock-logo.png" alt="sherlock"/></a>
   <br>
   <span>Hunt down social media accounts by username across <a href="https://sherlockproject.xyz/sites">400+ social networks</a></span>
   <br>
@@ -15,25 +15,27 @@
 </p>
 
 <p align="center">
-<img width="70%" height="70%" src="images/demo.png"/>
-</a>
+<img width="70%" height="70%" src="images/demo.png" alt="demo"/>
 </p>
 
 
 ## Installation
 
+> [!WARNING]  
+> Packages for ParrotOS and Ubuntu 24.04, maintained by a third party, appear to be __broken__.  
+> Users of these systems should defer to pipx/pip or Docker.
 
-| | Command | Notes |
-| - | - | - |
-| PyPI | `pipx install sherlock-project` | `pip` may be used in place of `pipx` |
-| Docker | `docker pull sherlock/sherlock` | |
-| Debian family | `apt install sherlock` | Kali, Parrot, Debian Testing and Sid |
-| BlackArch | `pacman -S sherlock` |  |
-| Homebrew | `brew install sherlock` | |
+| Method | Notes |
+| - | - |
+| `pipx install sherlock-project` | `pip` may be used in place of `pipx` |
+| `docker run -it --rm sherlock/sherlock` |
+| `dnf install sherlock-project` | |
+
+Community-maintained packages are available for Debian (>= 13), Ubuntu (>= 22.10), Homebrew, Kali, and BlackArch. These packages are not directly supported or maintained by the Sherlock Project.
 
 See all alternative installation methods [here](https://sherlockproject.xyz/installation)
 
-## Usage
+## General usage
 
 To search for only one user:
 ```bash
@@ -95,15 +97,35 @@ optional arguments:
   --local, -l           Force the use of the local data.json file.
   --nsfw                Include checking of NSFW sites from default list.
 ```
+## Apify Actor Usage [![Sherlock Actor](https://apify.com/actor-badge?actor=netmilk/sherlock)](https://apify.com/netmilk/sherlock?fpr=sherlock)
+
+<a href="https://apify.com/netmilk/sherlock?fpr=sherlock"><img src="https://apify.com/ext/run-on-apify.png" alt="Run Sherlock Actor on Apify" width="176" height="39" /></a>
+
+You can run Sherlock in the cloud without installation using the [Sherlock Actor](https://apify.com/netmilk/sherlock?fpr=sherlock) on [Apify](https://apify.com?fpr=sherlock) free of charge.
+
+``` bash
+$ echo '{"usernames":["user123"]}' | apify call -so netmilk/sherlock
+[{
+  "username": "user123",
+  "links": [
+    "https://www.1337x.to/user/user123/",
+    ...
+  ]
+}]
+```
+
+Read more about the [Sherlock Actor](../.actor/README.md), including how to use it programmatically via the Apify [API](https://apify.com/netmilk/sherlock/api?fpr=sherlock), [CLI](https://docs.apify.com/cli/?fpr=sherlock) and [JS/TS and Python SDKs](https://docs.apify.com/sdk?fpr=sherlock).
 
 ## Credits
 
 Thank you to everyone who has contributed to Sherlock! ❤️
 
 <a href="https://github.com/sherlock-project/sherlock/graphs/contributors">
-  <img src="https://contrib.rocks/image?&columns=25&max=10000&&repo=sherlock-project/sherlock" noZoom />
+  <img src="https://contrib.rocks/image?&columns=25&max=10000&&repo=sherlock-project/sherlock" alt="contributors"/>
 </a>
 
+## Star history
+
 <picture>
   <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=sherlock-project/sherlock&type=Date&theme=dark" />
   <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=sherlock-project/sherlock&type=Date" />

docs/removed-sites.md

@@ -84,22 +84,6 @@ As of 2020-02-23, all usernames are reported as not existing.
   },
 ```
 
-## Fanpop
-
-As of 2020-02-23, all usernames are reported as not existing.
-
-```json
-  "fanpop": {
-    "errorType": "response_url",
-    "errorUrl": "http://www.fanpop.com/",
-    "rank": 9454,
-    "url": "http://www.fanpop.com/fans/{}",
-    "urlMain": "http://www.fanpop.com/",
-    "username_claimed": "blue",
-    "username_unclaimed": "noonewould_everusethis7"
-  },
-```
-
 ## Canva
 
 As of 2020-02-23, all usernames are reported as not existing.
@@ -1273,19 +1257,6 @@ As of 2022-05-1, FanCentro returns false positives. Will later in new version of
   },
 ```
 
-## Codeforces
-As og 2022-05-01, Codeforces returns false positives
-```json
-  "Codeforces": {
-    "errorType": "response_url",
-    "errorUrl": "https://codeforces.com/",
-    "url": "https://codeforces.com/profile/{}",
-    "urlMain": "https://www.codeforces.com/",
-    "username_claimed": "tourist",
-    "username_unclaimed": "noonewouldeverusethis789"
-  },
-```
-
 ## Smashcast
 As og 2022-05-01, Smashcast is down
 ```json
@@ -1919,3 +1890,108 @@ __2024-06-10 :__ Http request returns 403 forbidden, and tries to verify the con
     "username_claimed": "JennyKrafts"
   }
 ```
+
+## Alik.cz
+__2024-07-21 :__ Target is now BLACKLISTED from the default manifest due to the site recieving unnecessarily high traffic from Sherlock (by request of the site owners). This target is not permitted to be reactivited. Inclusion in unrelated manifests is not impacted, but it is discouraged.
+
+## 8tracks
+__2025-02-02 :__ Might be dead again. Nobody knows for sure.
+```json
+"8tracks": {
+    "errorType": "message",
+    "errorMsg": "\"available\":true",
+    "headers": {
+      "Accept-Language": "en-US,en;q=0.5"
+    },
+    "url": "https://8tracks.com/{}",
+    "urlProbe": "https://8tracks.com/users/check_username?login={}&format=jsonh",
+    "urlMain": "https://8tracks.com/",
+    "username_claimed": "blue"
+  }
+```
+
+## Shpock
+__2025-02-02 :__ Can likely be added back with a new endpoint (source username availability endpoint from mobile app reg flow?)
+```json
+"Shpock": {
+    "errorType": "status_code",
+    "url": "https://www.shpock.com/shop/{}/items",
+    "urlMain": "https://www.shpock.com/",
+    "username_claimed": "user"
+  }
+```
+
+## Twitch
+__2025-02-02 :__
+```json
+"Twitch": {
+    "errorType": "message",
+    "errorMsg": "components.availability-tracking.warn-unavailable.component",
+    "url": "https://www.twitch.tv/{}",
+    "urlMain": "https://www.twitch.tv/",
+    "urlProbe": "https://m.twitch.tv/{}",
+    "username_claimed": "jenny"
+  }
+```
+
+## Fiverr
+__2025-02-02 :__ Fiverr added CSRF protections that messed with this test
+```json
+"Fiverr": {
+    "errorMsg": "\"status\":\"success\"",
+    "errorType": "message",
+    "headers": {
+      "Content-Type": "application/json",
+      "Accept-Language": "en-US,en;q=0.9"
+    },
+    "regexCheck": "^[A-Za-z][A-Za-z\\d_]{5,14}$",
+    "request_method": "POST",
+    "request_payload": {
+      "username": "{}"
+    },
+    "url": "https://www.fiverr.com/{}",
+    "urlMain": "https://www.fiverr.com/",
+    "urlProbe": "https://www.fiverr.com/validate_username",
+    "username_claimed": "blueman"
+  }
+```
+
+## BabyRU
+__2025-02-02 :__ Just being problematic (possibly related to errorMsg encoding?)
+```json
+"babyRU": {
+    "errorMsg": [
+      "\u0421\u0442\u0440\u0430\u043d\u0438\u0446\u0430, \u043a\u043e\u0442\u043e\u0440\u0443\u044e \u0432\u044b \u0438\u0441\u043a\u0430\u043b\u0438, \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d\u0430",
+      "Доступ с вашего IP-адреса временно ограничен"
+    ],
+    "errorType": "message",
+    "url": "https://www.baby.ru/u/{}/",
+    "urlMain": "https://www.baby.ru/",
+    "username_claimed": "blue"
+  }
+```
+
+## v0.dev
+__2025-02-16 :__ Unsure if any way to view profiles exists now
+```json
+"v0.dev": {
+    "errorType": "message",
+    "errorMsg": "<title>v0 by Vercel</title>",
+    "url": "https://v0.dev/{}",
+    "urlMain": "https://v0.dev",
+    "username_claimed": "t3dotgg"
+  }
+```
+
+## TorrentGalaxy
+__2025-07-06 :__ Site appears to have gone offline in March and hasn't come back
+```json
+  "TorrentGalaxy": {
+    "errorMsg": "<title>TGx:Can't show details</title>",
+    "errorType": "message",
+    "regexCheck": "^[A-Za-z0-9]{3,15}$",
+    "url": "https://torrentgalaxy.to/profile/{}",
+    "urlMain": "https://torrentgalaxy.to/",
+    "username_claimed": "GalaxyRG"
+  },
+```

pyproject.toml

@@ -40,7 +40,7 @@ repository = "https://github.com/sherlock-project/sherlock"
 "Bug Tracker" = "https://github.com/sherlock-project/sherlock/issues"
 
 [tool.poetry.dependencies]
-python = "^3.8"
+python = "^3.9"
 certifi = ">=2019.6.16"
 colorama = "^0.4.1"
 PySocks = "^1.7.0"
@@ -48,8 +48,7 @@ requests = "^2.22.0"
 requests-futures = "^1.0.0"
 stem = "^1.8.0"
 torrequest = "^0.1.0"
-# pandas can likely be bumped up to ^2.0.0 after fc39 EOL
-pandas = ">=1.0.0,<3.0.0"
+pandas = "^2.2.1"
 openpyxl = "^3.0.10"
 
 [tool.poetry.extras]
@@ -57,6 +56,9 @@ tor = ["torrequest"]
 
 [tool.poetry.group.dev.dependencies]
 jsonschema = "^4.0.0"
+rstr = "^3.2.2"
+pytest = "^8.4.2"
+pytest-xdist = "^3.8.0"
 
 [tool.poetry.scripts]
 sherlock = 'sherlock_project.sherlock:main'

pytest.ini

@@ -1,4 +1,7 @@
 [pytest]
-addopts = --strict-markers
+addopts = --strict-markers -m "not validate_targets"
 markers =
     online: mark tests are requiring internet access.
+    validate_targets: mark tests for sweeping manifest validation (sends many requests).
+    validate_targets_fp: validate_targets, false positive tests only.
+    validate_targets_fn: validate_targets, false negative tests only.

sherlock/resources/data.schema.json

@@ -1,80 +0,0 @@
-{
-    "$schema": "https://json-schema.org/draft/2020-12/schema",
-    "title": "Sherlock Target Manifest",
-    "description": "Social media targets to probe for the existence of known usernames",
-    "type": "object",
-    "properties": {
-        "$schema": { "type": "string" }
-    },
-    "patternProperties": {
-        "^(?!\\$).*?$": {
-            "type": "object",
-            "description": "Target name and associated information (key should be human readable name)",
-            "required": [ "url", "urlMain", "errorType", "username_claimed" ],
-            "properties": {
-                "url": { "type": "string" },
-                "urlMain": { "type": "string" },
-                "urlProbe": { "type": "string" },
-                "username_claimed": { "type": "string" },
-                "regexCheck": { "type": "string" },
-                "isNSFW": { "type": "boolean" },
-                "headers": { "type": "object" },
-                "request_payload": { "type": "object" },
-                "__comment__": {
-                    "type": "string",
-                    "description": "Used to clarify important target information if (and only if) a commit message would not suffice.\nThis key should not be parsed anywhere within Sherlock."
-                },
-                "tags": {
-                    "oneOf": [
-                        { "$ref": "#/$defs/tag" },
-                        { "type": "array", "items": { "$ref": "#/$defs/tag" } }
-                    ]
-                },
-                "request_method": {
-                    "type": "string",
-                    "enum": [ "GET", "POST", "HEAD", "PUT" ]
-                },
-                "errorType": {
-                    "type": "string",
-                    "enum": [ "message", "response_url", "status_code" ]
-                },
-                "errorMsg": {
-                    "oneOf": [
-                        { "type": "string" },
-                        { "type": "array", "items": { "type": "string" } }
-                    ]
-                },
-                "errorCode": {
-                    "oneOf": [
-                        { "type": "integer" },
-                        { "type": "array", "items": { "type": "integer" } }
-                    ]
-                },
-                "errorUrl": { "type": "string" },
-                "response_url": { "type": "string" }
-            },
-            "dependencies": {
-                "errorMsg": {
-                    "properties" : { "errorType": { "const": "message" } }
-                },
-                "errorUrl": {
-                    "properties": { "errorType": { "const": "response_url" } }
-                },
-                "errorCode": {
-                    "properties": { "errorType": { "const": "status_code" } }
-                }
-            },
-            "if": { "properties": { "errorType": { "const": "message" } } },
-            "then": { "required": [ "errorMsg" ] },
-            "else": {
-                "if": { "properties": { "errorType": { "const": "response_url" } } },
-                "then": { "required": [ "errorUrl" ] }
-            },
-            "additionalProperties": false
-        }
-    },
-    "additionalProperties": false,
-    "$defs": {
-        "tag": { "type": "string", "enum": [ "adult", "gaming" ] }
-    }
-}

sherlock_project/__init__.py

@@ -10,6 +10,6 @@ import_error_test_var = None
 
 __shortname__   = "Sherlock"
 __longname__    = "Sherlock: Find Usernames Across Social Networks"
-__version__     = "0.15.0"
+__version__     = "0.16.0"
 
 forge_api_latest_release = "https://api.github.com/repos/sherlock-project/sherlock/releases/latest"

sherlock_project/__main__.py

@@ -14,8 +14,8 @@ if __name__ == "__main__":
     # Check if the user is using the correct version of Python
     python_version = sys.version.split()[0]
 
-    if sys.version_info < (3, 8):
-        print(f"Sherlock requires Python 3.8+\nYou are using Python {python_version}, which is not supported by Sherlock.")
+    if sys.version_info < (3, 9):
+        print(f"Sherlock requires Python 3.9+\nYou are using Python {python_version}, which is not supported by Sherlock.")
         sys.exit(1)
 
     from sherlock_project import sherlock

sherlock_project/resources/data.json

@@ -17,26 +17,12 @@
     "urlMain": "https://2Dimensions.com/",
     "username_claimed": "blue"
   },
-  "3dnews": {
-    "errorMsg": "\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u0437\u0430\u0440\u0435\u0433\u0438\u0441\u0442\u0440\u0438\u0440\u043e\u0432\u0430\u043d \u0438 \u043d\u0435 \u0438\u043c\u0435\u0435\u0442 \u043f\u0440\u043e\u0444\u0438\u043b\u044f \u0434\u043b\u044f \u043f\u0440\u043e\u0441\u043c\u043e\u0442\u0440\u0430.",
-    "errorType": "message",
-    "url": "http://forum.3dnews.ru/member.php?username={}",
-    "urlMain": "http://forum.3dnews.ru/",
-    "username_claimed": "red"
-  },
   "7Cups": {
     "errorType": "status_code",
     "url": "https://www.7cups.com/@{}",
     "urlMain": "https://www.7cups.com/",
     "username_claimed": "blue"
   },
-  "8tracks": {
-    "errorMsg": "This page has vanished",
-    "errorType": "message",
-    "url": "https://8tracks.com/{}",
-    "urlMain": "https://8tracks.com/",
-    "username_claimed": "blue"
-  },
   "9GAG": {
     "errorType": "status_code",
     "url": "https://www.9gag.com/u/{}",
@@ -72,13 +58,6 @@
     "urlMain": "https://admireme.vip/",
     "username_claimed": "DemiDevil"
   },
-  "Air Pilot Life": {
-    "errorMsg": "Oops! That page doesn\u2019t exist or is private",
-    "errorType": "message",
-    "url": "https://airlinepilot.life/u/{}",
-    "urlMain": "https://airlinepilot.life/",
-    "username_claimed": "chris"
-  },
   "Airbit": {
     "errorType": "status_code",
     "url": "https://airbit.com/{}",
@@ -91,12 +70,6 @@
     "urlMain": "https://www.airliners.net/",
     "username_claimed": "yushinlin"
   },
-  "Alik.cz": {
-    "errorType": "status_code",
-    "url": "https://www.alik.cz/u/{}",
-    "urlMain": "https://www.alik.cz/",
-    "username_claimed": "julian"
-  },
   "All Things Worn": {
     "errorMsg": "Sell Used Panties",
     "errorType": "message",
@@ -113,12 +86,6 @@
     "urlMain": "https://allmylinks.com/",
     "username_claimed": "blue"
   },
-  "Amino": {
-    "errorType": "status_code",
-    "url": "https://aminoapps.com/u/{}",
-    "urlMain": "https://aminoapps.com",
-    "username_claimed": "blue"
-  },
   "AniWorld": {
     "errorMsg": "Dieses Profil ist nicht verf\u00fcgbar",
     "errorType": "message",
@@ -165,7 +132,8 @@
     "__comment__": "'The resource could not be found' relates to archive downtime",
     "errorMsg": [
       "could not fetch an account with user item identifier",
-      "The resource could not be found"
+      "The resource could not be found",
+      "Internet Archive services are temporarily offline"
     ],
     "errorType": "message",
     "url": "https://archive.org/details/@{}",
@@ -191,13 +159,11 @@
     "urlMain": "https://ask.fedoraproject.org/",
     "username_claimed": "red"
   },
-  "AskFM": {
-    "errorMsg": "Well, apparently not anymore.",
-    "errorType": "message",
-    "regexCheck": "^[a-zA-Z0-9_]{3,40}$",
-    "url": "https://ask.fm/{}",
-    "urlMain": "https://ask.fm/",
-    "username_claimed": "blue"
+  "Atcoder": {
+    "errorType": "status_code",
+    "url": "https://atcoder.jp/users/{}",
+    "urlMain": "https://atcoder.jp/",
+    "username_claimed": "ksun48"
   },
   "Audiojungle": {
     "errorType": "status_code",
@@ -219,17 +185,10 @@
     "urlMain": "https://www.avizo.cz/",
     "username_claimed": "blue"
   },
-  "BLIP.fm": {
-    "errorType": "status_code",
-    "regexCheck": "^[a-zA-Z0-9_]{1,30}$",
-    "url": "https://blip.fm/{}",
-    "urlMain": "https://blip.fm/",
-    "username_claimed": "blue"
-  },
   "BOOTH": {
     "errorType": "response_url",
     "errorUrl": "https://booth.pm/",
-    "regexCheck": "^[a-zA-Z0-9@_-]$",
+    "regexCheck": "^[\\w@-]+?$",
     "url": "https://{}.booth.pm/",
     "urlMain": "https://booth.pm/",
     "username_claimed": "blue"
@@ -265,12 +224,6 @@
     "urlMain": "https://www.biggerpockets.com/",
     "username_claimed": "blue"
   },
-  "Bikemap": {
-    "errorType": "status_code",
-    "url": "https://www.bikemap.net/en/u/{}/routes/created/",
-    "urlMain": "https://www.bikemap.net/",
-    "username_claimed": "bikemap"
-  },
   "BioHacking": {
     "errorType": "status_code",
     "url": "https://forum.dangerousthings.com/u/{}",
@@ -297,6 +250,12 @@
     "urlMain": "https://www.blipfoto.com/",
     "username_claimed": "blue"
   },
+  "Blitz Tactics": {
+    "errorType": "status_code",
+    "url": "https://blitztactics.com/{}",
+    "urlMain": "https://blitztactics.com/",
+    "username_claimed": "Lance5500"
+  },
   "Blogger": {
     "errorType": "status_code",
     "regexCheck": "^[a-zA-Z][a-zA-Z0-9_-]*$",
@@ -304,11 +263,19 @@
     "urlMain": "https://www.blogger.com/",
     "username_claimed": "blue"
   },
-  "BodyBuilding": {
-    "errorType": "response_url",
-    "errorUrl": "https://bodyspace.bodybuilding.com/",
-    "url": "https://bodyspace.bodybuilding.com/{}",
-    "urlMain": "https://bodyspace.bodybuilding.com/",
+  "Bluesky": {
+    "errorType": "status_code",
+    "url": "https://bsky.app/profile/{}.bsky.social",
+    "urlProbe": "https://public.api.bsky.app/xrpc/app.bsky.actor.getProfile?actor={}.bsky.social",
+    "urlMain": "https://bsky.app/",
+    "username_claimed": "mcuban"
+  },
+  "BoardGameGeek": {
+    "errorType": "message",
+    "regexCheck": "^[a-zA-Z0-9_]*$",
+    "errorMsg": "User not found",
+    "url": "https://boardgamegeek.com/user/{}",
+    "urlMain": "https://boardgamegeek.com",
     "username_claimed": "blue"
   },
   "BongaCams": {
@@ -391,7 +358,7 @@
   "Carbonmade": {
     "errorType": "response_url",
     "errorUrl": "https://carbonmade.com/fourohfour?domain={}.carbonmade.com",
-    "regexCheck": "^[a-zA-Z0-9@_-]$",
+    "regexCheck": "^[\\w@-]+?$",
     "url": "https://{}.carbonmade.com",
     "urlMain": "https://carbonmade.com/",
     "username_claimed": "jenny"
@@ -403,6 +370,12 @@
     "urlMain": "https://career.habr.com/",
     "username_claimed": "blue"
   },
+  "CashApp": {
+    "errorType": "status_code",
+    "url": "https://cash.app/${}",
+    "urlMain": "https://cash.app",
+    "username_claimed": "hotdiggitydog"
+  },
   "Championat": {
     "errorType": "status_code",
     "url": "https://www.championat.com/user/{}",
@@ -565,7 +538,7 @@
   },
   "Crevado": {
     "errorType": "status_code",
-    "regexCheck": "^[a-zA-Z0-9@_-]$",
+    "regexCheck": "^[\\w@-]+?$",
     "url": "https://{}.crevado.com",
     "urlMain": "https://crevado.com/",
     "username_claimed": "blue"
@@ -634,12 +607,38 @@
     "urlMain": "https://deviantart.com",
     "username_claimed": "blue"
   },
+  "DigitalSpy": {
+      "errorMsg": "The page you were looking for could not be found.",
+      "errorType": "message",
+      "url": "https://forums.digitalspy.com/profile/{}",
+      "urlMain": "https://forums.digitalspy.com/",
+      "username_claimed": "blue",
+      "regexCheck": "^\\w{3,20}$"
+  },
   "Discogs": {
     "errorType": "status_code",
     "url": "https://www.discogs.com/user/{}",
     "urlMain": "https://www.discogs.com/",
     "username_claimed": "blue"
   },
+  "Discord": {
+    "errorType": "message",
+    "url": "https://discord.com",
+    "urlMain": "https://discord.com/",
+    "urlProbe": "https://discord.com/api/v9/unique-username/username-attempt-unauthed",
+    "errorMsg": [
+      "{\"taken\":false}",
+      "The resource is being rate limited"
+    ],
+    "request_method": "POST",
+    "request_payload": {
+      "username": "{}"
+    },
+    "headers": {
+      "Content-Type": "application/json"
+    },
+    "username_claimed": "blue"
+  },
   "Discuss.Elastic.co": {
     "errorType": "status_code",
     "url": "https://discuss.elastic.co/u/{}",
@@ -670,9 +669,7 @@
   "Duolingo": {
     "errorMsg": "{\"users\":[]}",
     "errorType": "message",
-    "headers": {
-      "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/116.0"
-    },
+
     "url": "https://www.duolingo.com/profile/{}",
     "urlMain": "https://duolingo.com/",
     "urlProbe": "https://www.duolingo.com/2017-06-30/users?username={}",
@@ -685,6 +682,13 @@
     "urlMain": "https://community.eintracht.de/",
     "username_claimed": "mmammu"
   },
+  "Empretienda AR": {
+    "__comment__": "Note that Error Connecting responses may be indicative of unclaimed handles",
+    "errorType": "status_code",
+    "url": "https://{}.empretienda.com.ar",
+    "urlMain": "https://empretienda.com",
+    "username_claimed": "camalote"
+  },
   "Envato Forum": {
     "errorType": "status_code",
     "url": "https://forums.envato.com/u/{}",
@@ -700,10 +704,17 @@
   },
   "Exposure": {
     "errorType": "status_code",
+    "regexCheck": "^[a-zA-Z0-9-]{1,63}$",
     "url": "https://{}.exposure.co/",
     "urlMain": "https://exposure.co/",
     "username_claimed": "jonasjacobsson"
   },
+  "exophase": {
+    "errorType": "status_code",
+    "url": "https://www.exophase.com/user/{}/",
+    "urlMain": "https://www.exophase.com/",
+    "username_claimed": "blue"
+  },
   "EyeEm": {
     "errorType": "status_code",
     "url": "https://www.eyeem.com/u/{}",
@@ -728,25 +739,19 @@
     "urlMain": "https://www.fandom.com/",
     "username_claimed": "Jungypoo"
   },
+  "Fanpop": {
+    "errorType": "response_url",
+    "errorUrl": "https://www.fanpop.com/",
+    "url": "https://www.fanpop.com/fans/{}",
+    "urlMain": "https://www.fanpop.com/",
+    "username_claimed": "blue"
+  },
   "Finanzfrage": {
     "errorType": "status_code",
     "url": "https://www.finanzfrage.net/nutzer/{}",
     "urlMain": "https://www.finanzfrage.net/",
     "username_claimed": "finanzfrage"
   },
-  "Fiverr": {
-    "errorMsg": "\"status\":\"success\"",
-    "errorType": "message",
-    "regexCheck": "^[A-Za-z][A-Za-z\\d_]{5,14}$",
-    "request_method": "POST",
-    "request_payload": {
-      "username": "{}"
-    },
-    "url": "https://www.fiverr.com/{}",
-    "urlMain": "https://www.fiverr.com/",
-    "urlProbe": "https://www.fiverr.com/validate_username",
-    "username_claimed": "blueman"
-  },
   "Flickr": {
     "errorType": "status_code",
     "url": "https://www.flickr.com/people/{}",
@@ -795,6 +800,13 @@
     "urlMain": "https://fosstodon.org/",
     "username_claimed": "blue"
   },
+  "Framapiaf": {
+    "errorType": "status_code",
+    "regexCheck": "^[a-zA-Z0-9_]{1,30}$",
+    "url": "https://framapiaf.org/@{}",
+    "urlMain": "https://framapiaf.org",
+    "username_claimed": "pylapp"
+  },
   "Freelance.habr": {
     "errorMsg": "<div class=\"icon_user_locked\"></div>",
     "errorType": "message",
@@ -877,14 +889,15 @@
     "username_claimed": "bob"
   },
   "Giphy": {
-    "errorType": "status_code",
+    "errorType": "message",
+    "errorMsg": "<title> GIFs - Find &amp; Share on GIPHY</title>",
     "url": "https://giphy.com/{}",
     "urlMain": "https://giphy.com/",
-    "username_claimed": "blue"
+    "username_claimed": "red"
   },
   "GitBook": {
     "errorType": "status_code",
-    "regexCheck": "^[a-zA-Z0-9@_-]$",
+    "regexCheck": "^[\\w@-]+?$",
     "url": "https://{}.gitbook.io/",
     "urlMain": "https://gitbook.com/",
     "username_claimed": "gitbook"
@@ -904,6 +917,12 @@
     "urlProbe": "https://gitlab.com/api/v4/users?username={}",
     "username_claimed": "blue"
   },
+  "Gitea": {
+    "errorType": "status_code",
+    "url": "https://gitea.com/{}",
+    "urlMain": "https://gitea.com/",
+    "username_claimed": "xorm"
+  },
   "Gitee": {
     "errorType": "status_code",
     "url": "https://gitee.com/{}",
@@ -960,8 +979,8 @@
   },
   "HackTheBox": {
     "errorType": "status_code",
-    "url": "https://forum.hackthebox.eu/profile/{}",
-    "urlMain": "https://forum.hackthebox.eu/",
+    "url": "https://forum.hackthebox.com/u/{}",
+    "urlMain": "https://forum.hackthebox.com/",
     "username_claimed": "angar"
   },
   "Hackaday": {
@@ -971,7 +990,7 @@
     "username_claimed": "adam"
   },
   "HackenProof (Hackers)": {
-    "errorMsg": "<title>Web3\u2019s Largest Ethical Hackers Community | HackenProof</title>",
+    "errorMsg": "Page not found",
     "errorType": "message",
     "regexCheck": "^[\\w-]{,34}$",
     "url": "https://hackenproof.com/hackers/{}",
@@ -1044,8 +1063,7 @@
     "username_claimed": "red"
   },
   "Houzz": {
-    "errorMsg": "The page you requested was not found.",
-    "errorType": "message",
+    "errorType": "status_code",
     "url": "https://houzz.com/user/{}",
     "urlMain": "https://houzz.com/",
     "username_claimed": "blue"
@@ -1064,17 +1082,17 @@
     "username_claimed": "blue"
   },
   "HudsonRock": {
-    "errorMsg": "No results",
+    "errorMsg": "This username is not associated",
     "errorType": "message",
     "url": "https://cavalier.hudsonrock.com/api/json/v2/osint-tools/search-by-username?username={}",
     "urlMain": "https://hudsonrock.com",
     "username_claimed": "testadmin"
   },
-  "ICQ": {
+  "Hugging Face": {
     "errorType": "status_code",
-    "url": "https://icq.im/{}/en",
-    "urlMain": "https://icq.com/",
-    "username_claimed": "Micheal"
+    "url": "https://huggingface.co/{}",
+    "urlMain": "https://huggingface.co/",
+    "username_claimed": "Pasanlaksitha"
   },
   "IFTTT": {
     "errorType": "status_code",
@@ -1121,7 +1139,7 @@
     "errorType": "status_code",
     "url": "https://instagram.com/{}",
     "urlMain": "https://instagram.com/",
-    "urlProbe": "https://www.picuki.com/profile/{}",
+    "urlProbe": "https://imginn.com/{}",
     "username_claimed": "instagram"
   },
   "Instructables": {
@@ -1154,7 +1172,7 @@
   },
   "Itch.io": {
     "errorType": "status_code",
-    "regexCheck": "^[a-zA-Z0-9@_-]$",
+    "regexCheck": "^[\\w@-]+?$",
     "url": "https://{}.itch.io/",
     "urlMain": "https://itch.io/",
     "username_claimed": "blue"
@@ -1175,7 +1193,7 @@
   },
   "Jimdo": {
     "errorType": "status_code",
-    "regexCheck": "^[a-zA-Z0-9@_-]$",
+    "regexCheck": "^[\\w@-]+?$",
     "url": "https://{}.jimdosite.com",
     "urlMain": "https://jimdosite.com/",
     "username_claimed": "jenny"
@@ -1186,18 +1204,18 @@
     "urlMain": "https://discourse.joplinapp.org/",
     "username_claimed": "laurent"
   },
-  "KEAKR": {
-    "errorType": "status_code",
-    "url": "https://www.keakr.com/en/profile/{}",
-    "urlMain": "https://www.keakr.com/",
-    "username_claimed": "beats"
-  },
   "Kaggle": {
     "errorType": "status_code",
     "url": "https://www.kaggle.com/{}",
     "urlMain": "https://www.kaggle.com/",
     "username_claimed": "dansbecker"
   },
+  "kaskus": {
+    "errorType": "status_code",
+    "url": "https://www.kaskus.co.id/@{}",
+    "urlMain": "https://www.kaskus.co.id/",
+    "username_claimed": "l0mbart"
+  },
   "Keybase": {
     "errorType": "status_code",
     "url": "https://keybase.io/{}",
@@ -1206,8 +1224,7 @@
   },
   "Kick": {
     "__comment__": "Cloudflare. Only viable when proxied.",
-    "errorMsg": "Not Found",
-    "errorType": "message",
+    "errorType": "status_code",
     "url": "https://kick.com/{}",
     "urlMain": "https://kick.com/",
     "urlProbe": "https://kick.com/api/v2/channels/{}",
@@ -1224,8 +1241,7 @@
   "Kongregate": {
     "errorType": "status_code",
     "headers": {
-      "Accept": "text/html",
-      "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/116.0"
+      "Accept": "text/html"
     },
     "regexCheck": "^[a-zA-Z][a-zA-Z0-9_-]*$",
     "url": "https://www.kongregate.com/accounts/{}",
@@ -1264,24 +1280,23 @@
     "username_claimed": "blue"
   },
   "LibraryThing": {
-    "errorMsg": "Catalog your books online",
+    "errorMsg": "<p>Error: This user doesn't exist</p>",
     "errorType": "message",
+    "headers": {
+      "Cookie": "LTAnonSessionID=3159599315; LTUnifiedCookie=%7B%22areyouhuman%22%3A1%7D; "
+    },
     "url": "https://www.librarything.com/profile/{}",
     "urlMain": "https://www.librarything.com/",
     "username_claimed": "blue"
   },
   "Lichess": {
-    "errorMsg": "Page not found!",
-    "errorType": "message",
+    "errorType": "status_code",
     "url": "https://lichess.org/@/{}",
     "urlMain": "https://lichess.org",
-    "username_claimed": "blue"
+    "username_claimed": "john"
   },
   "LinkedIn": {
     "errorType": "status_code",
-    "headers": {
-      "User-Agent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/W.X.Y.Z Safari/537.36"
-    },
     "regexCheck": "^[a-zA-Z0-9]{3,100}$",
     "request_method": "GET",
     "url": "https://linkedin.com/in/{}",
@@ -1296,6 +1311,12 @@
     "urlMain": "https://linktr.ee/",
     "username_claimed": "anne"
   },
+  "LinuxFR.org": {
+    "errorType": "status_code",
+    "url": "https://linuxfr.org/users/{}",
+    "urlMain": "https://linuxfr.org/",
+    "username_claimed": "pylapp"
+  },
   "Listed": {
     "errorType": "response_url",
     "errorUrl": "https://listed.to/@{}",
@@ -1336,12 +1357,12 @@
     "urlMain": "https://forums.mmorpg.com/",
     "username_claimed": "goku"
   },
-  "Mapify": {
-    "errorType": "response_url",
-    "errorUrl": "https://mapify.travel/{}",
-    "url": "https://mapify.travel/{}",
-    "urlMain": "https://mapify.travel/",
-    "username_claimed": "mapify"
+  "Mamot": {
+    "errorType": "status_code",
+    "regexCheck": "^[a-zA-Z0-9_]{1,30}$",
+    "url": "https://mamot.fr/@{}",
+    "urlMain": "https://mamot.fr/",
+    "username_claimed": "anciensEnssat"
   },
   "Medium": {
     "errorMsg": "<body",
@@ -1358,8 +1379,8 @@
     "username_claimed": "blue"
   },
   "Minecraft": {
-    "errorCode": 204,
-    "errorType": "status_code",
+    "errorMsg": "Couldn't find any profile with name",
+    "errorType": "message",
     "url": "https://api.mojang.com/users/profiles/minecraft/{}",
     "urlMain": "https://minecraft.net/",
     "username_claimed": "blue"
@@ -1504,11 +1525,12 @@
     "urlMain": "https://nyaa.si/",
     "username_claimed": "blue"
   },
-  "OGUsers": {
-    "errorType": "status_code",
-    "url": "https://ogu.gg/{}",
-    "urlMain": "https://ogu.gg/",
-    "username_claimed": "ogusers"
+  "Open Collective": {
+    "errorMsg": "Oops! Page not found",
+    "errorType": "message",
+    "url": "https://opencollective.com/{}",
+    "urlMain": "https://opencollective.com/",
+    "username_claimed": "pylapp"
   },
   "OpenStreetMap": {
     "errorType": "status_code",
@@ -1530,6 +1552,13 @@
     "urlMain": "https://ourdjtalk.com/",
     "username_claimed": "steve"
   },
+  "Outgress": {
+    "errorMsg": "Outgress - Error",
+    "errorType": "message",
+    "url": "https://outgress.com/agents/{}",
+    "urlMain": "https://outgress.com/",
+    "username_claimed": "pylapp"
+  },  
   "PCGamer": {
     "errorMsg": "The specified member cannot be found. Please enter a member's entire name.",
     "errorType": "message",
@@ -1591,12 +1620,31 @@
     "urlMain": "https://www.pinkbike.com/",
     "username_claimed": "blue"
   },
+  "pixelfed.social": {
+    "errorType": "status_code",
+    "url": "https://pixelfed.social/{}/",
+    "urlMain": "https://pixelfed.social",
+    "username_claimed": "pylapp"
+  },
   "PlayStore": {
     "errorType": "status_code",
     "url": "https://play.google.com/store/apps/developer?id={}",
     "urlMain": "https://play.google.com/store",
     "username_claimed": "Facebook"
   },
+  "Playstrategy": {
+    "errorType": "status_code",
+    "url": "https://playstrategy.org/@/{}",
+    "urlMain": "https://playstrategy.org",
+    "username_claimed": "oruro"
+  },
+  "Plurk": {
+    "errorMsg": "User Not Found!",
+    "errorType": "message",
+    "url": "https://www.plurk.com/{}",
+    "urlMain": "https://www.plurk.com/",
+    "username_claimed": "plurkoffice"
+  },
   "PocketStars": {
     "errorMsg": "Join Your Favorite Adult Stars",
     "errorType": "message",
@@ -1639,12 +1687,25 @@
     "username_claimed": "blue"
   },
   "ProductHunt": {
-    "errorMsg": "We seem to have lost this page",
-    "errorType": "message",
+    "errorType": "status_code",
     "url": "https://www.producthunt.com/@{}",
     "urlMain": "https://www.producthunt.com/",
     "username_claimed": "jenny"
   },
+  "programming.dev": {
+    "errorMsg": "Error!",
+    "errorType": "message",
+    "url": "https://programming.dev/u/{}",
+    "urlMain": "https://programming.dev",
+    "username_claimed": "pylapp"
+  },
+  "Pychess": {
+  "errorType": "message",
+  "errorMsg": "404",
+  "url": "https://www.pychess.org/@/{}",
+  "urlMain": "https://www.pychess.org",
+  "username_claimed": "gbtami"
+  },
   "PromoDJ": {
     "errorType": "status_code",
     "url": "http://promodj.com/{}",
@@ -1654,16 +1715,23 @@
   "PyPi": {
     "errorType": "status_code",
     "url": "https://pypi.org/user/{}",
+    "urlProbe": "https://pypi.org/_includes/administer-user-include/{}",
     "urlMain": "https://pypi.org",
     "username_claimed": "Blue"
   },
   "Rajce.net": {
     "errorType": "status_code",
-    "regexCheck": "^[a-zA-Z0-9@_-]$",
+    "regexCheck": "^[\\w@-]+?$",
     "url": "https://{}.rajce.idnes.cz/",
     "urlMain": "https://www.rajce.idnes.cz/",
     "username_claimed": "blue"
   },
+  "Rarible": {
+    "errorType": "status_code",
+    "url": "https://rarible.com/marketplace/api/v4/urls/{}",
+    "urlMain": "https://rarible.com/",
+    "username_claimed": "blue"
+  },
   "Rate Your Music": {
     "errorType": "status_code",
     "url": "https://rateyourmusic.com/~{}",
@@ -1800,12 +1868,6 @@
     "urlMain": "https://www.shitpostbot.com/",
     "username_claimed": "blue"
   },
-  "Shpock": {
-    "errorType": "status_code",
-    "url": "https://www.shpock.com/shop/{}/items",
-    "urlMain": "https://www.shpock.com/",
-    "username_claimed": "user"
-  },
   "Signal": {
     "errorMsg": "Oops! That page doesn\u2019t exist or is private.",
     "errorType": "message",
@@ -1841,7 +1903,8 @@
     "username_claimed": "blue"
   },
   "SlideShare": {
-    "errorType": "status_code",
+    "errorType": "message",
+    "errorMsg": "<title>Username available</title>",
     "url": "https://slideshare.net/{}",
     "urlMain": "https://slideshare.net/",
     "username_claimed": "blue"
@@ -1855,6 +1918,7 @@
   },
   "SmugMug": {
     "errorType": "status_code",
+    "regexCheck": "^[a-zA-Z]{1,35}$",
     "url": "https://{}.smugmug.com",
     "urlMain": "https://smugmug.com",
     "username_claimed": "winchester"
@@ -1893,12 +1957,17 @@
     "urlMain": "https://soylentnews.org",
     "username_claimed": "adam"
   },
+  "SpeakerDeck": {
+    "errorType": "status_code",
+    "url": "https://speakerdeck.com/{}",
+    "urlMain": "https://speakerdeck.com/",
+    "username_claimed": "pylapp"
+  },
   "Speedrun.com": {
-    "errorMsg": "Not found",
-    "errorType": "message",
-    "url": "https://speedrun.com/user/{}",
+    "errorType": "status_code",
+    "url": "https://speedrun.com/users/{}",
     "urlMain": "https://speedrun.com/",
-    "username_claimed": "3Tau"
+    "username_claimed": "example"
   },
   "Spells8": {
     "errorType": "status_code",
@@ -1939,9 +2008,7 @@
   },
   "Spotify": {
     "errorType": "status_code",
-    "headers": {
-      "user-agent": "PostmanRuntime/7.29.2"
-    },
+
     "url": "https://open.spotify.com/user/{}",
     "urlMain": "https://open.spotify.com/",
     "username_claimed": "blue"
@@ -1968,8 +2035,7 @@
     "username_claimed": "blue"
   },
   "Strava": {
-    "errorMsg": "Strava | Running, Cycling &amp; Hiking App - Train, Track &amp; Share",
-    "errorType": "message",
+    "errorType": "status_code",
     "regexCheck": "^[^.]*?$",
     "url": "https://www.strava.com/athletes/{}",
     "urlMain": "https://www.strava.com/",
@@ -1989,12 +2055,19 @@
     "urlProbe": "https://ch.tetr.io/api/users/{}",
     "username_claimed": "osk"
   },
-  "TLDR Legal": {
+  "Tiendanube": {
+    "url": "https://{}.mitiendanube.com/",
+    "urlMain": "https://www.tiendanube.com/",
     "errorType": "status_code",
-    "regexCheck": "^[a-zA-Z0-9]{3,20}$",
-    "url": "https://tldrlegal.com/users/{}/",
-    "urlMain": "https://tldrlegal.com/",
-    "username_claimed": "kevin"
+    "username_claimed": "blue"
+  },
+  "Topcoder": {
+    "errorType": "status_code",
+    "url": "https://profiles.topcoder.com/{}/",
+    "urlMain": "https://topcoder.com/",
+    "username_claimed": "USER",
+    "urlProbe": "https://api.topcoder.com/v5/members/{}",
+    "regexCheck": "[a-zA-Z0-9 ]"
   },
   "TRAKTRAIN": {
     "errorType": "status_code",
@@ -2039,14 +2112,6 @@
     "urlMain": "https://www.tnaflix.com/",
     "username_claimed": "hacker"
   },
-  "TorrentGalaxy": {
-    "errorMsg": "<title>TGx:Can't show details</title>",
-    "errorType": "message",
-    "regexCheck": "^[A-Za-z0-9]{3,15}$",
-    "url": "https://torrentgalaxy.to/profile/{}",
-    "urlMain": "https://torrentgalaxy.to/",
-    "username_claimed": "GalaxyRG"
-  },
   "TradingView": {
     "errorType": "status_code",
     "request_method": "GET",
@@ -2104,20 +2169,16 @@
     "urlMain": "https://tweakers.net",
     "username_claimed": "femme"
   },
-  "Twitch": {
-    "errorType": "status_code",
-    "url": "https://www.twitch.tv/{}",
-    "urlMain": "https://www.twitch.tv/",
-    "urlProbe": "https://m.twitch.tv/{}",
-    "username_claimed": "jenny"
-  },
   "Twitter": {
-    "errorMsg": "<div class=\"error-panel\"><span>User ",
+    "errorMsg": [
+      "<div class=\"error-panel\"><span>User ",
+      "<title>429 Too Many Requests</title>"
+    ],
     "errorType": "message",
     "regexCheck": "^[a-zA-Z0-9_]{1,15}$",
     "url": "https://x.com/{}",
     "urlMain": "https://x.com/",
-    "urlProbe": "https://nitter.net/{}",
+    "urlProbe": "https://nitter.privacydev.net/{}",
     "username_claimed": "blue"
   },
   "Typeracer": {
@@ -2159,6 +2220,12 @@
     "urlMain": "https://vsco.co/",
     "username_claimed": "blue"
   },
+  "Velog": {
+    "errorType": "status_code",
+    "url": "https://velog.io/@{}/posts",
+    "urlMain": "https://velog.io/",
+    "username_claimed": "qlgks1"
+  },
   "Velomania": {
     "errorMsg": "\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u0437\u0430\u0440\u0435\u0433\u0438\u0441\u0442\u0440\u0438\u0440\u043e\u0432\u0430\u043d \u0438 \u043d\u0435 \u0438\u043c\u0435\u0435\u0442 \u043f\u0440\u043e\u0444\u0438\u043b\u044f \u0434\u043b\u044f \u043f\u0440\u043e\u0441\u043c\u043e\u0442\u0440\u0430.",
     "errorType": "message",
@@ -2180,7 +2247,8 @@
     "username_claimed": "jenny"
   },
   "Vero": {
-    "errorType": "status_code",
+    "errorMsg": "Not Found",
+    "errorType": "message",
     "request_method": "GET",
     "url": "https://vero.co/{}",
     "urlMain": "https://vero.co/",
@@ -2200,6 +2268,12 @@
     "urlProbe": "https://www.virustotal.com/ui/users/{}/avatar",
     "username_claimed": "blue"
   },
+  "VLR": {
+    "errorType": "status_code",
+    "url": "https://www.vlr.gg/user/{}",
+    "urlMain": "https://www.vlr.gg",
+    "username_claimed": "optms"
+  },
   "WICG Forum": {
     "errorType": "status_code",
     "regexCheck": "^(?![.-])[a-zA-Z0-9_.-]{3,20}$",
@@ -2222,7 +2296,7 @@
   },
   "WebNode": {
     "errorType": "status_code",
-    "regexCheck": "^[a-zA-Z0-9@_-]$",
+    "regexCheck": "^[\\w@-]+?$",
     "url": "https://{}.webnode.cz/",
     "urlMain": "https://www.webnode.cz/",
     "username_claimed": "radkabalcarova"
@@ -2236,6 +2310,7 @@
   },
   "Weebly": {
     "errorType": "status_code",
+    "regexCheck": "^[a-zA-Z0-9-]{1,63}$",
     "url": "https://{}.weebly.com/",
     "urlMain": "https://weebly.com/",
     "username_claimed": "blue"
@@ -2262,7 +2337,7 @@
   },
   "Wix": {
     "errorType": "status_code",
-    "regexCheck": "^[a-zA-Z0-9@_-]$",
+    "regexCheck": "^[\\w@-]+?$",
     "url": "https://{}.wix.com",
     "urlMain": "https://wix.com/",
     "username_claimed": "support"
@@ -2350,9 +2425,7 @@
   },
   "YouTube": {
     "errorType": "status_code",
-    "headers": {
-      "User-Agent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/W.X.Y.Z Safari/537.36"
-    },
+
     "url": "https://www.youtube.com/@{}",
     "urlMain": "https://www.youtube.com/",
     "username_claimed": "youtube"
@@ -2369,13 +2442,6 @@
     "urlMain": "http://www.authorstream.com/",
     "username_claimed": "blue"
   },
-  "babyRU": {
-    "errorMsg": "\u0421\u0442\u0440\u0430\u043d\u0438\u0446\u0430, \u043a\u043e\u0442\u043e\u0440\u0443\u044e \u0432\u044b \u0438\u0441\u043a\u0430\u043b\u0438, \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d\u0430",
-    "errorType": "message",
-    "url": "https://www.baby.ru/u/{}/",
-    "urlMain": "https://www.baby.ru/",
-    "username_claimed": "blue"
-  },
   "babyblogRU": {
     "errorType": "response_url",
     "errorUrl": "https://www.babyblog.ru/",
@@ -2480,12 +2546,6 @@
     "urlMain": "https://www.geocaching.com/",
     "username_claimed": "blue"
   },
-  "gfycat": {
-    "errorType": "status_code",
-    "url": "https://gfycat.com/@{}",
-    "urlMain": "https://gfycat.com/",
-    "username_claimed": "Test"
-  },
   "habr": {
     "errorType": "status_code",
     "url": "https://habr.com/ru/users/{}",
@@ -2505,13 +2565,6 @@
     "urlMain": "https://www.hunting.ru/forum/",
     "username_claimed": "red"
   },
-  "iMGSRC.RU": {
-    "errorType": "response_url",
-    "errorUrl": "https://imgsrc.ru/",
-    "url": "https://imgsrc.ru/main/user.php?user={}",
-    "urlMain": "https://imgsrc.ru/",
-    "username_claimed": "blue"
-  },
   "igromania": {
     "errorMsg": "\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u0437\u0430\u0440\u0435\u0433\u0438\u0441\u0442\u0440\u0438\u0440\u043e\u0432\u0430\u043d \u0438 \u043d\u0435 \u0438\u043c\u0435\u0435\u0442 \u043f\u0440\u043e\u0444\u0438\u043b\u044f \u0434\u043b\u044f \u043f\u0440\u043e\u0441\u043c\u043e\u0442\u0440\u0430.",
     "errorType": "message",
@@ -2589,12 +2642,6 @@
     "urlMain": "https://chaos.social/",
     "username_claimed": "Gargron"
   },
-  "mastodon.technology": {
-    "errorType": "status_code",
-    "url": "https://mastodon.technology/@{}",
-    "urlMain": "https://mastodon.xyz/",
-    "username_claimed": "ashfurrow"
-  },
   "mastodon.xyz": {
     "errorType": "status_code",
     "url": "https://mastodon.xyz/@{}",
@@ -2635,7 +2682,7 @@
   },
   "nnRU": {
     "errorType": "status_code",
-    "regexCheck": "^[a-zA-Z0-9@_-]$",
+    "regexCheck": "^[\\w@-]+?$",
     "url": "https://{}.www.nn.ru/",
     "urlMain": "https://www.nn.ru/",
     "username_claimed": "blue"
@@ -2652,6 +2699,14 @@
     "urlMain": "https://www.npmjs.com/",
     "username_claimed": "kennethsweezy"
   },
+  "omg.lol": {
+    "errorMsg": "\"available\": true",
+    "errorType": "message",
+    "url": "https://{}.omg.lol",
+    "urlMain": "https://home.omg.lol",
+    "urlProbe": "https://api.omg.lol/address/{}/availability",
+    "username_claimed": "adam"
+  },
   "opennet": {
     "errorMsg": "\u0418\u043c\u044f \u0443\u0447\u0430\u0441\u0442\u043d\u0438\u043a\u0430 \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d\u043e",
     "errorType": "message",
@@ -2693,12 +2748,6 @@
     "urlMain": "https://prog.hu/",
     "username_claimed": "Sting"
   },
-  "queer.af": {
-    "errorType": "status_code",
-    "url": "https://queer.af/@{}",
-    "urlMain": "https://queer.af/",
-    "username_claimed": "erincandescent"
-  },
   "satsisRU": {
     "errorType": "status_code",
     "url": "https://satsis.info/user/{}",
@@ -2711,13 +2760,6 @@
     "urlMain": "https://sessionize.com/",
     "username_claimed": "jason-mayes"
   },
-  "skyrock": {
-    "errorType": "status_code",
-    "regexCheck": "^[a-zA-Z0-9@_-]$",
-    "url": "https://{}.skyrock.com/",
-    "urlMain": "https://skyrock.com/",
-    "username_claimed": "red"
-  },
   "social.tchncs.de": {
     "errorType": "status_code",
     "url": "https://social.tchncs.de/@{}",
@@ -2736,23 +2778,39 @@
     "urlMain": "https://www.svidbook.ru/",
     "username_claimed": "green"
   },
+  "threads": {
+    "errorMsg": "<title>Threads</title>",
+    "errorType": "message",
+    "headers": {
+      "Sec-Fetch-Mode": "navigate"
+    },
+    "url": "https://www.threads.net/@{}",
+    "urlMain": "https://www.threads.net/",
+    "username_claimed": "zuck"
+  },
   "toster": {
     "errorType": "status_code",
     "url": "https://www.toster.ru/user/{}/answers",
     "urlMain": "https://www.toster.ru/",
     "username_claimed": "adam"
   },
+  "tumblr": {
+    "errorType": "status_code",
+    "url": "https://{}.tumblr.com/",
+    "urlMain": "https://www.tumblr.com/",
+    "username_claimed": "goku"
+},
   "uid": {
     "errorType": "status_code",
     "url": "http://uid.me/{}",
     "urlMain": "https://uid.me/",
     "username_claimed": "blue"
   },
-  "wiki.vg": {
+  "write.as": {
     "errorType": "status_code",
-    "url": "https://wiki.vg/User:{}",
-    "urlMain": "https://wiki.vg/",
-    "username_claimed": "Auri"
+    "url": "https://write.as/{}",
+    "urlMain": "https://write.as",
+    "username_claimed": "pylapp"
   },
   "xHamster": {
     "errorType": "status_code",
@@ -2767,5 +2825,20 @@
     "url": "https://www.znanylekarz.pl/{}",
     "urlMain": "https://znanylekarz.pl",
     "username_claimed": "janusz-nowak"
+  },
+  "Bluesky": {
+    "errorType": "status_code",
+    "url": "https://bsky.app/profile/{}.bsky.social",
+    "urlProbe": "https://public.api.bsky.app/xrpc/app.bsky.actor.getProfile?actor={}.bsky.social",
+    "urlMain": "https://bsky.app/",
+    "username_claimed": "mcuban"
+  },
+  "Platzi": {
+    "errorType": "status_code",
+    "errorCode": 404,
+    "url": "https://platzi.com/p/{}/",
+    "urlMain": "https://platzi.com/",
+    "username_claimed": "freddier",
+    "request_method": "GET"
   }
 }

sherlock_project/sherlock.py

@@ -24,6 +24,7 @@ import re
 from argparse import ArgumentParser, RawDescriptionHelpFormatter
 from json import loads as json_loads
 from time import monotonic
+from typing import Optional
 
 import requests
 from requests_futures.sessions import FuturesSession
@@ -167,15 +168,15 @@ def multiple_usernames(username):
 
 
 def sherlock(
-    username,
-    site_data,
+    username: str,
+    site_data: dict[str, dict[str, str]],
     query_notify: QueryNotify,
     tor: bool = False,
     unique_tor: bool = False,
     dump_response: bool = False,
-    proxy=None,
-    timeout=60,
-):
+    proxy: Optional[str] = None,
+    timeout: int = 60,
+) -> dict[str, dict[str, str | QueryResult]]:
     """Run Sherlock Analysis.
 
     Checks for existence of username on various social media sites.
@@ -261,7 +262,7 @@ def sherlock(
         # A user agent is needed because some sites don't return the correct
         # information since they think that we are bots (Which we actually are...)
         headers = {
-            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/116.0",
+            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:129.0) Gecko/20100101 Firefox/129.0",
         }
 
         if "headers" in net_info:
@@ -412,8 +413,10 @@ def sherlock(
         # be highly targetted. Comment at the end of each fingerprint to
         # indicate target and date fingerprinted.
         WAFHitMsgs = [
-            '.loading-spinner{visibility:hidden}body.no-js .challenge-running{display:none}body.dark{background-color:#222;color:#d9d9d9}body.dark a{color:#fff}body.dark a:hover{color:#ee730a;text-decoration:underline}body.dark .lds-ring div{border-color:#999 transparent transparent}body.dark .font-red{color:#b20f03}body.dark', # 2024-05-13 Cloudflare
-            '{return l.onPageView}}),Object.defineProperty(r,"perimeterxIdentifiers",{enumerable:' # 2024-04-09 PerimeterX / Human Security
+            r'.loading-spinner{visibility:hidden}body.no-js .challenge-running{display:none}body.dark{background-color:#222;color:#d9d9d9}body.dark a{color:#fff}body.dark a:hover{color:#ee730a;text-decoration:underline}body.dark .lds-ring div{border-color:#999 transparent transparent}body.dark .font-red{color:#b20f03}body.dark', # 2024-05-13 Cloudflare
+            r'<span id="challenge-error-text">', # 2024-11-11 Cloudflare error page
+            r'AwsWafIntegration.forceRefreshToken', # 2024-11-11 Cloudfront (AWS)
+            r'{return l.onPageView}}),Object.defineProperty(r,"perimeterxIdentifiers",{enumerable:' # 2024-04-09 PerimeterX / Human Security
         ]
 
         if error_text is not None:
@@ -474,7 +477,7 @@ def sherlock(
             raise ValueError(
                 f"Unknown Error Type '{error_type}' for " f"site '{social_network}'"
             )
-        
+
         if dump_response:
             print("+++++++++++++++++++++")
             print(f"TARGET NAME   : {social_network}")
@@ -504,7 +507,7 @@ def sherlock(
             print("+++++++++++++++++++++")
 
         # Notify caller about results of query.
-        result = QueryResult(
+        result: QueryResult = QueryResult(
             username=username,
             site_name=social_network,
             site_url_user=url,
@@ -653,7 +656,7 @@ def main():
         metavar="JSON_FILE",
         dest="json_file",
         default=None,
-        help="Load data from a JSON file or an online, valid, JSON file.",
+        help="Load data from a JSON file or an online, valid, JSON file. Upstream PR numbers also accepted.",
     )
     parser.add_argument(
         "--timeout",
@@ -716,6 +719,22 @@ def main():
         help="Include checking of NSFW sites from default list.",
     )
 
+    parser.add_argument(
+        "--no-txt",
+        action="store_true",
+        dest="no_txt",
+        default=False,
+        help="Disable creation of a txt file",
+    )
+
+    parser.add_argument(
+        "--ignore-exclusions",
+        action="store_true",
+        dest="ignore_exclusions",
+        default=False,
+        help="Ignore upstream exclusions (may return more false positives)",
+    )
+
     args = parser.parse_args()
 
     # If the user presses CTRL-C, exit gracefully without throwing errors
@@ -773,10 +792,32 @@ def main():
     try:
         if args.local:
             sites = SitesInformation(
-                os.path.join(os.path.dirname(__file__), "resources/data.json")
+                os.path.join(os.path.dirname(__file__), "resources/data.json"),
+                honor_exclusions=False,
             )
         else:
-            sites = SitesInformation(args.json_file)
+            json_file_location = args.json_file
+            if args.json_file:
+                # If --json parameter is a number, interpret it as a pull request number
+                if args.json_file.isnumeric():
+                    pull_number = args.json_file
+                    pull_url = f"https://api.github.com/repos/sherlock-project/sherlock/pulls/{pull_number}"
+                    pull_request_raw = requests.get(pull_url).text
+                    pull_request_json = json_loads(pull_request_raw)
+
+                    # Check if it's a valid pull request
+                    if "message" in pull_request_json:
+                        print(f"ERROR: Pull request #{pull_number} not found.")
+                        sys.exit(1)
+
+                    head_commit_sha = pull_request_json["head"]["sha"]
+                    json_file_location = f"https://raw.githubusercontent.com/sherlock-project/sherlock/{head_commit_sha}/sherlock_project/resources/data.json"
+
+            sites = SitesInformation(
+                data_file_path=json_file_location,
+                honor_exclusions=not args.ignore_exclusions,
+                do_not_exclude=args.site_list,
+            )
     except Exception as error:
         print(f"ERROR:  {error}")
         sys.exit(1)
@@ -847,14 +888,15 @@ def main():
         else:
             result_file = f"{username}.txt"
 
-        with open(result_file, "w", encoding="utf-8") as file:
-            exists_counter = 0
-            for website_name in results:
-                dictionary = results[website_name]
-                if dictionary.get("status").status == QueryStatus.CLAIMED:
-                    exists_counter += 1
-                    file.write(dictionary["url_user"] + "\n")
-            file.write(f"Total Websites Username Detected On : {exists_counter}\n")
+        if not args.no_txt:
+            with open(result_file, "w", encoding="utf-8") as file:
+                exists_counter = 0
+                for website_name in results:
+                    dictionary = results[website_name]
+                    if dictionary.get("status").status == QueryStatus.CLAIMED:
+                        exists_counter += 1
+                        file.write(dictionary["url_user"] + "\n")
+                file.write(f"Total Websites Username Detected On : {exists_counter}\n")
 
         if args.csv:
             result_file = f"{username}.csv"

sherlock_project/sites.py

@@ -7,6 +7,10 @@ import json
 import requests
 import secrets
 
+
+MANIFEST_URL = "https://raw.githubusercontent.com/sherlock-project/sherlock/master/sherlock_project/resources/data.json"
+EXCLUSIONS_URL = "https://raw.githubusercontent.com/sherlock-project/sherlock/refs/heads/exclusions/false_positive_exclusions.txt"
+
 class SiteInformation:
     def __init__(self, name, url_home, url_username_format, username_claimed,
                 information, is_nsfw, username_unclaimed=secrets.token_urlsafe(10)):
@@ -67,12 +71,17 @@ class SiteInformation:
         Return Value:
         Nicely formatted string to get information about this object.
         """
-        
+
         return f"{self.name} ({self.url_home})"
 
 
 class SitesInformation:
-    def __init__(self, data_file_path=None):
+    def __init__(
+            self,
+            data_file_path: str|None = None,
+            honor_exclusions: bool = True,
+            do_not_exclude: list[str] = [],
+        ):
         """Create Sites Information Object.
 
         Contains information about all supported websites.
@@ -110,7 +119,7 @@ class SitesInformation:
             # The default data file is the live data.json which is in the GitHub repo. The reason why we are using
             # this instead of the local one is so that the user has the most up-to-date data. This prevents
             # users from creating issue about false positives which has already been fixed or having outdated data
-            data_file_path = "https://raw.githubusercontent.com/sherlock-project/sherlock/master/sherlock_project/resources/data.json"
+            data_file_path = MANIFEST_URL
 
         # Ensure that specified data file has correct extension.
         if not data_file_path.lower().endswith(".json"):
@@ -152,9 +161,31 @@ class SitesInformation:
                 raise FileNotFoundError(f"Problem while attempting to access "
                                         f"data file '{data_file_path}'."
                                         )
-        
+
         site_data.pop('$schema', None)
 
+        if honor_exclusions:
+            try:
+                response = requests.get(url=EXCLUSIONS_URL)
+                if response.status_code == 200:
+                    exclusions = response.text.splitlines()
+                    exclusions = [exclusion.strip() for exclusion in exclusions]
+
+                    for site in do_not_exclude:
+                        if site in exclusions:
+                            exclusions.remove(site)
+
+                    for exclusion in exclusions:
+                        try:
+                            site_data.pop(exclusion, None)
+                        except KeyError:
+                            pass
+
+            except Exception:
+                # If there was any problem loading the exclusions, just continue without them
+                print("Warning: Could not load exclusions, continuing without them.")
+                honor_exclusions = False
+
         self.sites = {}
 
         # Add all site information from the json file to internal site list.
@@ -194,7 +225,7 @@ class SitesInformation:
         for site in self.sites:
             if self.sites[site].is_nsfw and site.casefold() not in do_not_remove:
                 continue
-            sites[site] = self.sites[site]  
+            sites[site] = self.sites[site]
         self.sites =  sites
 
     def site_name_list(self):

tests/conftest.py

@@ -4,6 +4,11 @@ import urllib
 import pytest
 from sherlock_project.sites import SitesInformation
 
+def fetch_local_manifest() -> dict[str, dict[str, str]]:
+    sites_obj = SitesInformation(data_file_path=os.path.join(os.path.dirname(__file__), "../sherlock_project/resources/data.json"))
+    sites_iterable = {site.name: site.information for site in sites_obj}
+    return sites_iterable
+
 @pytest.fixture()
 def sites_obj():
     sites_obj = SitesInformation(data_file_path=os.path.join(os.path.dirname(__file__), "../sherlock_project/resources/data.json"))
@@ -11,9 +16,7 @@ def sites_obj():
 
 @pytest.fixture(scope="session")
 def sites_info():
-    sites_obj = SitesInformation(data_file_path=os.path.join(os.path.dirname(__file__), "../sherlock_project/resources/data.json"))
-    sites_iterable = {site.name: site.information for site in sites_obj}
-    yield sites_iterable
+    yield fetch_local_manifest()
 
 @pytest.fixture(scope="session")
 def remote_schema():
@@ -21,3 +24,10 @@ def remote_schema():
     with urllib.request.urlopen(schema_url) as remoteschema:
         schemadat = json.load(remoteschema)
     yield schemadat
+
+def pytest_generate_tests(metafunc):
+    if "chunked_sites" in metafunc.fixturenames:
+        sites_info = fetch_local_manifest()
+        params = [{name: data} for name, data in sites_info.items()]
+        ids = list(sites_info.keys())
+        metafunc.parametrize("chunked_sites", params, ids=ids)

tests/sherlock_interactives.py

@@ -7,8 +7,8 @@ class Interactives:
     def run_cli(args:str = "") -> str:
         """Pass arguments to Sherlock as a normal user on the command line"""
         # Adapt for platform differences (Windows likes to be special)
-        if platform.system == "Windows":
-            command:str = f"py -m sherlock {args}"
+        if platform.system() == "Windows":
+            command:str = f"py -m sherlock_project {args}"
         else:
             command:str = f"sherlock {args}"
 
@@ -20,8 +20,7 @@ class Interactives:
             raise InteractivesSubprocessError(e.output.decode())
 
 
-    # -> list[str] is prefered, but will require deprecation of support for Python 3.8
-    def walk_sherlock_for_files_with(pattern: str) -> list:
+    def walk_sherlock_for_files_with(pattern: str) -> list[str]:
         """Check all files within the Sherlock package for matching patterns"""
         pattern:re.Pattern = re.compile(pattern)
         matching_files:list[str] = []

tests/test_manifest.py

@@ -7,7 +7,7 @@ def test_validate_manifest_against_local_schema():
     """Ensures that the manifest matches the local schema, for situations where the schema is being changed."""
     json_relative: str = '../sherlock_project/resources/data.json'
     schema_relative: str = '../sherlock_project/resources/data.schema.json'
-    
+
     json_path: str = os.path.join(os.path.dirname(__file__), json_relative)
     schema_path: str = os.path.join(os.path.dirname(__file__), schema_relative)
 

tests/test_probes.py

@@ -44,7 +44,7 @@ class TestLiveTargets:
 
     # Known positives should only use sites trusted to be reliable and unchanging
     @pytest.mark.parametrize('site,username',[
-        ('BodyBuilding', 'blue'),
+        ('Keybase', 'blue'),
         ('devRant', 'blue'),
     ])
     def test_known_positives_via_response_url(self, sites_info, site, username):

tests/test_validate_targets.py

@@ -0,0 +1,99 @@
+import pytest
+import re
+import rstr
+
+from sherlock_project.sherlock import sherlock
+from sherlock_project.notify import QueryNotify
+from sherlock_project.result import QueryResult, QueryStatus
+
+
+FALSE_POSITIVE_ATTEMPTS: int = 2    # Since the usernames are randomly generated, it's POSSIBLE that a real username can be hit
+FALSE_POSITIVE_QUANTIFIER_UPPER_BOUND: int = 15  # If a pattern uses quantifiers such as `+` `*` or `{n,}`, limit the upper bound (0 to disable)
+FALSE_POSITIVE_DEFAULT_PATTERN: str = r'^[a-zA-Z0-9]{7,20}$'  # Used in absence of a regexCheck entry
+
+
+def set_pattern_upper_bound(pattern: str, upper_bound: int = FALSE_POSITIVE_QUANTIFIER_UPPER_BOUND) -> str:
+    """Set upper bound for regex patterns that use quantifiers such as `+` `*` or `{n,}`."""
+    def replace_upper_bound(match: re.Match) -> str: # type: ignore
+        lower_bound: int = int(match.group(1)) if match.group(1) else 0 # type: ignore
+        upper_bound = upper_bound if lower_bound < upper_bound else lower_bound # type: ignore  # noqa: F823
+        return f'{{{lower_bound},{upper_bound}}}'
+
+    pattern = re.sub(r'(?<!\\)\{(\d+),\}', replace_upper_bound, pattern) # {n,} # type: ignore
+    pattern = re.sub(r'(?<!\\)\+', f'{{1,{upper_bound}}}', pattern) # +
+    pattern = re.sub(r'(?<!\\)\*', f'{{0,{upper_bound}}}', pattern) # *
+
+    return pattern
+
+def false_positive_check(sites_info: dict[str, dict[str, str]], site: str, pattern: str) -> QueryStatus:
+    """Check if a site is likely to produce false positives."""
+    status: QueryStatus = QueryStatus.UNKNOWN
+
+    for _ in range(FALSE_POSITIVE_ATTEMPTS):
+        query_notify: QueryNotify = QueryNotify()
+        username: str = rstr.xeger(pattern)
+
+        result: QueryResult | str = sherlock(
+            username=username,
+            site_data=sites_info,
+            query_notify=query_notify,
+        )[site]['status']
+
+        if not hasattr(result, 'status'):
+            raise TypeError(f"Result for site {site} does not have 'status' attribute. Actual result: {result}")
+        if type(result.status) is not QueryStatus: # type: ignore
+            raise TypeError(f"Result status for site {site} is not of type QueryStatus. Actual type: {type(result.status)}") # type: ignore
+        status = result.status # type: ignore
+
+        if status in (QueryStatus.AVAILABLE, QueryStatus.WAF):
+            return status
+
+    return status
+
+
+def false_negative_check(sites_info: dict[str, dict[str, str]], site: str) -> QueryStatus:
+    """Check if a site is likely to produce false negatives."""
+    status: QueryStatus = QueryStatus.UNKNOWN
+    query_notify: QueryNotify = QueryNotify()
+
+    result: QueryResult | str = sherlock(
+        username=sites_info[site]['username_claimed'],
+        site_data=sites_info,
+        query_notify=query_notify,
+    )[site]['status']
+
+    if not hasattr(result, 'status'):
+            raise TypeError(f"Result for site {site} does not have 'status' attribute. Actual result: {result}")
+    if type(result.status) is not QueryStatus: # type: ignore
+        raise TypeError(f"Result status for site {site} is not of type QueryStatus. Actual type: {type(result.status)}") # type: ignore
+    status = result.status # type: ignore
+
+    return status
+
+@pytest.mark.validate_targets
+@pytest.mark.online
+class Test_All_Targets:
+
+    @pytest.mark.validate_targets_fp
+    def test_false_pos(self, chunked_sites: dict[str, dict[str, str]]):
+        """Iterate through all sites in the manifest to discover possible false-positive inducting targets."""
+        pattern: str
+        for site in chunked_sites:
+            try:
+                pattern = chunked_sites[site]['regexCheck']
+            except KeyError:
+                pattern = FALSE_POSITIVE_DEFAULT_PATTERN
+
+            if FALSE_POSITIVE_QUANTIFIER_UPPER_BOUND > 0:
+                pattern = set_pattern_upper_bound(pattern)
+
+            result: QueryStatus = false_positive_check(chunked_sites, site, pattern)
+            assert result is QueryStatus.AVAILABLE, f"{site} produced false positive with pattern {pattern}, result was {result}"
+
+    @pytest.mark.validate_targets_fn
+    def test_false_neg(self, chunked_sites: dict[str, dict[str, str]]):
+        """Iterate through all sites in the manifest to discover possible false-negative inducting targets."""
+        for site in chunked_sites:
+            result: QueryStatus = false_negative_check(chunked_sites, site)
+            assert result is QueryStatus.CLAIMED, f"{site} produced false negative, result was {result}"
+

tox.ini

@@ -7,8 +7,6 @@ envlist =
     py312
     py311
     py310
-    py39
-    py38
 
 [testenv]
 description = Attempt to build and install the package
@@ -16,6 +14,7 @@ deps =
     coverage
     jsonschema
     pytest
+    rstr
 allowlist_externals = coverage
 commands =
     coverage run --source=sherlock_project --module pytest -v
@@ -37,8 +36,7 @@ commands =
 
 [gh-actions]
 python =
+    3.13: py313
     3.12: py312
     3.11: py311
     3.10: py310
-    3.9: py39
-    3.8: py38