mirror of
https://github.com/masterking32/MasterHttpRelayVPN.git
synced 2026-05-17 21:24:37 +03:00
Merge branch 'python_testing' into features/google-ip-scanner
This commit is contained in:
@@ -45,12 +45,42 @@ This means the filter sees normal-looking Google traffic, while the actual desti
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Step-by-Step Setup Guide
|
## Quick Start (Recommended)
|
||||||
|
|
||||||
|
One command sets up a virtualenv, installs dependencies, launches an interactive
|
||||||
|
config wizard, and starts the proxy.
|
||||||
|
|
||||||
|
**Windows:**
|
||||||
|
```cmd
|
||||||
|
git clone https://github.com/masterking32/MasterHttpRelayVPN.git
|
||||||
|
cd MasterHttpRelayVPN
|
||||||
|
start.bat
|
||||||
|
```
|
||||||
|
|
||||||
|
**Linux / macOS:**
|
||||||
|
```bash
|
||||||
|
git clone https://github.com/masterking32/MasterHttpRelayVPN.git
|
||||||
|
cd MasterHttpRelayVPN
|
||||||
|
chmod +x start.sh
|
||||||
|
./start.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
The first time it runs, the wizard asks for your Google Apps Script Deployment ID
|
||||||
|
and generates a strong random password for you. Follow the Apps Script deployment
|
||||||
|
instructions in **Step 2** below before running the wizard so you have a
|
||||||
|
Deployment ID ready.
|
||||||
|
|
||||||
|
After it's running, jump to **Step 5** (browser proxy) and **Step 6** (CA
|
||||||
|
certificate).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Step-by-Step Setup Guide (Manual)
|
||||||
|
|
||||||
### Step 1: Download This Project
|
### Step 1: Download This Project
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone -b python_testing https://github.com/masterking32/MasterHttpRelayVPN.git
|
git clone https://github.com/masterking32/MasterHttpRelayVPN.git
|
||||||
cd MasterHttpRelayVPN
|
cd MasterHttpRelayVPN
|
||||||
pip install -r requirements.txt
|
pip install -r requirements.txt
|
||||||
```
|
```
|
||||||
@@ -60,7 +90,7 @@ pip install -r requirements.txt
|
|||||||
> pip install -r requirements.txt -i https://mirror-pypi.runflare.com/simple/ --trusted-host mirror-pypi.runflare.com
|
> pip install -r requirements.txt -i https://mirror-pypi.runflare.com/simple/ --trusted-host mirror-pypi.runflare.com
|
||||||
> ```
|
> ```
|
||||||
|
|
||||||
Or download the ZIP from [GitHub](https://github.com/masterking32/MasterHttpRelayVPN/tree/python_testing) and extract it.
|
Or download the ZIP from [GitHub](https://github.com/masterking32/MasterHttpRelayVPN) and extract it.
|
||||||
|
|
||||||
### Step 2: Set Up the Google Relay (Code.gs)
|
### Step 2: Set Up the Google Relay (Code.gs)
|
||||||
|
|
||||||
@@ -86,6 +116,15 @@ This is the "relay" that sits on Google's servers and fetches websites for you.
|
|||||||
|
|
||||||
### Step 3: Configure
|
### Step 3: Configure
|
||||||
|
|
||||||
|
**Option A — interactive wizard (recommended):**
|
||||||
|
```bash
|
||||||
|
python setup.py
|
||||||
|
```
|
||||||
|
It'll prompt for your Deployment ID, generate a random `auth_key`, and write
|
||||||
|
`config.json` for you.
|
||||||
|
|
||||||
|
**Option B — manual:**
|
||||||
|
|
||||||
1. Copy the example config file:
|
1. Copy the example config file:
|
||||||
```bash
|
```bash
|
||||||
cp config.example.json config.json
|
cp config.example.json config.json
|
||||||
@@ -174,6 +213,29 @@ Firefox uses its own certificate store, so even after OS-level install you need
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
## LAN Sharing (Optional)
|
||||||
|
|
||||||
|
By default, the proxy only listens on `127.0.0.1` (localhost), meaning only your computer can use it. To allow other devices on your local network (LAN) to use the proxy:
|
||||||
|
|
||||||
|
1. Set `"lan_sharing": true` in your `config.json`
|
||||||
|
2. The proxy will automatically listen on all network interfaces (`0.0.0.0`)
|
||||||
|
3. The startup log will show your LAN IP addresses that other devices can connect to
|
||||||
|
|
||||||
|
**Example LAN configuration:**
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"lan_sharing": true,
|
||||||
|
"listen_host": "0.0.0.0",
|
||||||
|
"listen_port": 8085
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Security Warning:** When LAN sharing is enabled, anyone on your local network can use your proxy. Ensure your network is trusted and consider additional security measures.
|
||||||
|
|
||||||
|
**On other devices:** Configure them to use your computer's LAN IP (shown in the startup log) and port 8085 as the HTTP proxy.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## Modes Overview
|
## Modes Overview
|
||||||
|
|
||||||
This project focuses entirely on the **Apps Script** relay — a free Google account is all you need, no server, no VPS, no Cloudflare setup. Everything is configured out of the box for this mode.
|
This project focuses entirely on the **Apps Script** relay — a free Google account is all you need, no server, no VPS, no Cloudflare setup. Everything is configured out of the box for this mode.
|
||||||
@@ -188,8 +250,9 @@ This project focuses entirely on the **Apps Script** relay — a free Google acc
|
|||||||
|---------|-------------|
|
|---------|-------------|
|
||||||
| `auth_key` | Password shared between your computer and the relay |
|
| `auth_key` | Password shared between your computer and the relay |
|
||||||
| `script_id` | Your Google Apps Script Deployment ID |
|
| `script_id` | Your Google Apps Script Deployment ID |
|
||||||
| `listen_host` | Where to listen (`127.0.0.1` = only this computer) |
|
| `listen_host` | Where to listen (`127.0.0.1` = only this computer, `0.0.0.0` = all interfaces for LAN sharing) |
|
||||||
| `listen_port` | Which port to listen on (default: `8085`) |
|
| `listen_port` | Which port to listen on (default: `8085`) |
|
||||||
|
| `lan_sharing` | Enable LAN sharing to allow other devices on your network to use the proxy (`false` by default) |
|
||||||
| `log_level` | How much detail to show: `DEBUG`, `INFO`, `WARNING`, `ERROR` |
|
| `log_level` | How much detail to show: `DEBUG`, `INFO`, `WARNING`, `ERROR` |
|
||||||
|
|
||||||
### Advanced Settings
|
### Advanced Settings
|
||||||
@@ -215,6 +278,7 @@ Install everything from [`requirements.txt`](requirements.txt). All listed packa
|
|||||||
| `h2` | HTTP/2 multiplexing to the Apps Script relay (significantly faster) |
|
| `h2` | HTTP/2 multiplexing to the Apps Script relay (significantly faster) |
|
||||||
| `brotli` | Decompression of `Content-Encoding: br` responses |
|
| `brotli` | Decompression of `Content-Encoding: br` responses |
|
||||||
| `zstandard` | Decompression of `Content-Encoding: zstd` responses |
|
| `zstandard` | Decompression of `Content-Encoding: zstd` responses |
|
||||||
|
| `netifaces` | Better network interface detection for LAN sharing (fallback available without it) |
|
||||||
|
|
||||||
### Load Balancing
|
### Load Balancing
|
||||||
|
|
||||||
@@ -229,7 +293,7 @@ To increase speed, deploy `Code.gs` multiple times to different Apps Script proj
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
> ⚠️ **Note:** If you are using multiple deployments, the auth-keys must be identical. (All deployments must use the same auth-key.)
|
||||||
---
|
---
|
||||||
|
|
||||||
## Updating the Google Relay
|
## Updating the Google Relay
|
||||||
@@ -316,8 +380,10 @@ After scanning, update your `config.json` with the recommended IP and restart th
|
|||||||
```
|
```
|
||||||
MasterHttpRelayVPN/
|
MasterHttpRelayVPN/
|
||||||
├── main.py # Entry point: starts the proxy
|
├── main.py # Entry point: starts the proxy
|
||||||
|
├── setup.py # Interactive wizard — writes config.json
|
||||||
|
├── start.bat / start.sh # One-click launcher (venv + deps + wizard + run)
|
||||||
├── config.example.json # Copy to config.json and fill in your values
|
├── config.example.json # Copy to config.json and fill in your values
|
||||||
├── requirements.txt # Optional Python dependencies
|
├── requirements.txt # Python dependencies
|
||||||
├── apps_script/
|
├── apps_script/
|
||||||
│ └── Code.gs # The relay script you deploy to Google Apps Script
|
│ └── Code.gs # The relay script you deploy to Google Apps Script
|
||||||
├── ca/ # Generated MITM CA (do NOT share)
|
├── ca/ # Generated MITM CA (do NOT share)
|
||||||
@@ -360,7 +426,7 @@ MasterHttpRelayVPN/
|
|||||||
- **Change the default `AUTH_KEY`** in `Code.gs` before deploying.
|
- **Change the default `AUTH_KEY`** in `Code.gs` before deploying.
|
||||||
- **Don't share the `ca/` folder** — it contains your private certificate key.
|
- **Don't share the `ca/` folder** — it contains your private certificate key.
|
||||||
- Keep `listen_host` as `127.0.0.1` so only your computer can use the proxy.
|
- Keep `listen_host` as `127.0.0.1` so only your computer can use the proxy.
|
||||||
|
- Every google scripts deployment has limit of 20,000 requests in 24 hours
|
||||||
---
|
---
|
||||||
|
|
||||||
## Special Thanks
|
## Special Thanks
|
||||||
|
|||||||
+33
-7
@@ -162,6 +162,7 @@ Firefox معمولا certificate store جداگانه دارد:
|
|||||||
|
|
||||||
نکته امنیتی: پوشه `ca/` را با کسی به اشتراک نگذارید. اگر خواستید از اول گواهی جدید بسازید، این پوشه را حذف کنید تا دوباره ساخته شود.
|
نکته امنیتی: پوشه `ca/` را با کسی به اشتراک نگذارید. اگر خواستید از اول گواهی جدید بسازید، این پوشه را حذف کنید تا دوباره ساخته شود.
|
||||||
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## حالتهای موجود
|
## حالتهای موجود
|
||||||
@@ -170,15 +171,38 @@ Firefox معمولا certificate store جداگانه دارد:
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## تنظیمات مهم
|
## اشتراکگذاری در شبکه محلی (اختیاری)
|
||||||
|
|
||||||
|
بهطور پیشفرض، پروکسی فقط به `127.0.0.1` (localhost) گوش میدهد، به این معنی که فقط کامپیوتر شما میتواند از آن استفاده کند. برای اینکه سایر دستگاههای موجود در شبکه محلی (LAN) شما بتوانند از این پروکسی استفاده کنند:
|
||||||
|
|
||||||
|
۱. در فایل `config.json` خود، مقدار `"lan_sharing"` را `true` قرار دهید.
|
||||||
|
۲. پروکسی به طور خودکار به تمام رابطهای شبکه (`0.0.0.0`) گوش خواهد داد.
|
||||||
|
۳. در لاگ راهاندازی، آدرسهای IP شبکه محلی شما که سایر دستگاهها میتوانند به آن متصل شوند، نمایش داده میشود.
|
||||||
|
|
||||||
|
**نمونه پیکربندی برای شبکه محلی:**
|
||||||
|
json
|
||||||
|
{
|
||||||
|
"lan_sharing": true,
|
||||||
|
"listen_host": "0.0.0.0",
|
||||||
|
"listen_port": 8085
|
||||||
|
}
|
||||||
|
|
||||||
|
**هشدار امنیتی:** وقتی اشتراکگذاری در شبکه محلی فعال باشد، هر کسی در شبکه محلی شما میتواند از پروکسی شما استفاده کند. اطمینان حاصل کنید که شبکه شما مورد اعتماد است و اقدامات امنیتی بیشتری را در نظر بگیرید.
|
||||||
|
|
||||||
|
**در سایر دستگاهها:** آنها را طوری پیکربندی کنید که از آدرس IP کامپیوتر شما در شبکه محلی (که در لاگ راهاندازی نمایش داده میشود) و پورت 8085 به عنوان پروکسی HTTP استفاده کنند.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## تنظیمات اصلی
|
||||||
|
|
||||||
| تنظیم | توضیح |
|
| تنظیم | توضیح |
|
||||||
|------|-------|
|
|------|-------|
|
||||||
| `auth_key` | رمز مشترک بین برنامه و رله |
|
| `auth_key` | رمز مشترک بین کامپیوتر شما و رله |
|
||||||
| `script_id` | Deployment ID مربوط به Apps Script |
|
| `script_id` | شناسه Deployment مربوط به Google Apps Script شما |
|
||||||
| `listen_host` | آدرس محلی برای اجرا |
|
| `listen_host` | محل گوش دادن (`127.0.0.1` = فقط همین کامپیوتر، `0.0.0.0` = همه اینترفیسها برای اشتراکگذاری LAN) |
|
||||||
| `listen_port` | پورت پراکسی |
|
| `listen_port` | پورتی که پروکسی روی آن اجرا میشود (پیشفرض: `8085`) |
|
||||||
| `log_level` | میزان جزئیات لاگ |
|
| `lan_sharing` | فعالسازی اشتراکگذاری LAN تا دستگاههای دیگر در شبکه شما بتوانند از پروکسی استفاده کنند (بهصورت پیشفرض `false`) |
|
||||||
|
| `log_level` | میزان جزئیات لاگ: `DEBUG`، `INFO`، `WARNING`، `ERROR` |
|
||||||
|
|
||||||
### تنظیمات پیشرفته
|
### تنظیمات پیشرفته
|
||||||
|
|
||||||
@@ -202,6 +226,7 @@ Firefox معمولا certificate store جداگانه دارد:
|
|||||||
| `h2` | ارتباط HTTP/2 با رله Apps Script (بهطور محسوسی سریعتر) |
|
| `h2` | ارتباط HTTP/2 با رله Apps Script (بهطور محسوسی سریعتر) |
|
||||||
| `brotli` | پشتیبانی از فشردهسازی `Content-Encoding: br` |
|
| `brotli` | پشتیبانی از فشردهسازی `Content-Encoding: br` |
|
||||||
| `zstandard` | پشتیبانی از فشردهسازی `Content-Encoding: zstd` |
|
| `zstandard` | پشتیبانی از فشردهسازی `Content-Encoding: zstd` |
|
||||||
|
| `netifaces` | تشخیص بهتر اینترفیسهای شبکه برای اشتراکگذاری LAN (در صورت نبود آن، حالت جایگزین در دسترس است) |
|
||||||
|
|
||||||
### استفاده از چند Script ID
|
### استفاده از چند Script ID
|
||||||
|
|
||||||
@@ -216,7 +241,7 @@ Firefox معمولا certificate store جداگانه دارد:
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
> **نکته :** اگر از چندین دیپلویمنت آیدی استفاده میکنید توجه داشته باشید که auth_key های همه دیپلویمنت ها باید یکسان باشند.
|
||||||
---
|
---
|
||||||
|
|
||||||
## بهروزرسانی `Code.gs`
|
## بهروزرسانی `Code.gs`
|
||||||
@@ -345,6 +370,7 @@ MasterHttpRelayVPN/
|
|||||||
- مقدار پیشفرض `AUTH_KEY` را قبل از deploy عوض کنید.
|
- مقدار پیشفرض `AUTH_KEY` را قبل از deploy عوض کنید.
|
||||||
- پوشه `ca/` را منتشر نکنید.
|
- پوشه `ca/` را منتشر نکنید.
|
||||||
- بهتر است `listen_host` روی `127.0.0.1` بماند.
|
- بهتر است `listen_host` روی `127.0.0.1` بماند.
|
||||||
|
- هر دیپلویمنت روی گوگل اسکریپت دارای محدودیت 20,000 درخواست در هر 24 ساعت است
|
||||||
---
|
---
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|||||||
+8
-1
@@ -10,6 +10,8 @@
|
|||||||
"socks5_port": 1080,
|
"socks5_port": 1080,
|
||||||
"log_level": "INFO",
|
"log_level": "INFO",
|
||||||
"verify_ssl": true,
|
"verify_ssl": true,
|
||||||
|
"lan_sharing": true,
|
||||||
|
"parallel_relay": 1,
|
||||||
"block_hosts": [],
|
"block_hosts": [],
|
||||||
"bypass_hosts": [
|
"bypass_hosts": [
|
||||||
"localhost",
|
"localhost",
|
||||||
@@ -29,7 +31,12 @@
|
|||||||
"calendar.google.com",
|
"calendar.google.com",
|
||||||
"drive.google.com",
|
"drive.google.com",
|
||||||
"docs.google.com",
|
"docs.google.com",
|
||||||
"chat.google.com"
|
"chat.google.com",
|
||||||
|
"maps.google.com",
|
||||||
|
"play.google.com",
|
||||||
|
"translate.google.com",
|
||||||
|
"assistant.google.com",
|
||||||
|
"lens.google.com"
|
||||||
],
|
],
|
||||||
"direct_google_allow": [
|
"direct_google_allow": [
|
||||||
"www.google.com",
|
"www.google.com",
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ if _SRC_DIR not in sys.path:
|
|||||||
|
|
||||||
from cert_installer import install_ca, is_ca_trusted
|
from cert_installer import install_ca, is_ca_trusted
|
||||||
from constants import __version__
|
from constants import __version__
|
||||||
|
from lan_utils import log_lan_access
|
||||||
from google_ip_scanner import scan_sync
|
from google_ip_scanner import scan_sync
|
||||||
from logging_utils import configure as configure_logging, print_banner
|
from logging_utils import configure as configure_logging, print_banner
|
||||||
from mitm import CA_CERT_FILE
|
from mitm import CA_CERT_FILE
|
||||||
@@ -109,8 +110,31 @@ def main():
|
|||||||
config = json.load(f)
|
config = json.load(f)
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
print(f"Config not found: {config_path}")
|
print(f"Config not found: {config_path}")
|
||||||
print("Copy config.example.json to config.json and fill in your values.")
|
# Offer the interactive wizard if it's available and we're on a TTY.
|
||||||
sys.exit(1)
|
wizard = os.path.join(os.path.dirname(os.path.abspath(__file__)), "setup.py")
|
||||||
|
if os.path.exists(wizard) and sys.stdin.isatty():
|
||||||
|
try:
|
||||||
|
answer = input("Run the interactive setup wizard now? [Y/n]: ").strip().lower()
|
||||||
|
except EOFError:
|
||||||
|
answer = "n"
|
||||||
|
if answer in ("", "y", "yes"):
|
||||||
|
import subprocess
|
||||||
|
rc = subprocess.call([sys.executable, wizard])
|
||||||
|
if rc != 0:
|
||||||
|
sys.exit(rc)
|
||||||
|
try:
|
||||||
|
with open(config_path) as f:
|
||||||
|
config = json.load(f)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Could not load config after setup: {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
else:
|
||||||
|
print("Copy config.example.json to config.json and fill in your values,")
|
||||||
|
print("or run: python setup.py")
|
||||||
|
sys.exit(1)
|
||||||
|
else:
|
||||||
|
print("Run: python setup.py (or copy config.example.json to config.json)")
|
||||||
|
sys.exit(1)
|
||||||
except json.JSONDecodeError as e:
|
except json.JSONDecodeError as e:
|
||||||
print(f"Invalid JSON in config: {e}")
|
print(f"Invalid JSON in config: {e}")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
@@ -220,6 +244,14 @@ def main():
|
|||||||
else:
|
else:
|
||||||
log.info("MITM CA is already trusted.")
|
log.info("MITM CA is already trusted.")
|
||||||
|
|
||||||
|
# ── LAN sharing configuration ────────────────────────────────────────
|
||||||
|
lan_sharing = config.get("lan_sharing", True)
|
||||||
|
if lan_sharing:
|
||||||
|
# If LAN sharing is enabled and host is still localhost, change to all interfaces
|
||||||
|
if config.get("listen_host", "127.0.0.1") == "127.0.0.1":
|
||||||
|
config["listen_host"] = "0.0.0.0"
|
||||||
|
log.info("LAN sharing enabled — listening on all interfaces")
|
||||||
|
|
||||||
log.info("HTTP proxy : %s:%d",
|
log.info("HTTP proxy : %s:%d",
|
||||||
config.get("listen_host", "127.0.0.1"),
|
config.get("listen_host", "127.0.0.1"),
|
||||||
config.get("listen_port", 8080))
|
config.get("listen_port", 8080))
|
||||||
@@ -228,6 +260,11 @@ def main():
|
|||||||
config.get("socks5_host", config.get("listen_host", "127.0.0.1")),
|
config.get("socks5_host", config.get("listen_host", "127.0.0.1")),
|
||||||
config.get("socks5_port", 1080))
|
config.get("socks5_port", 1080))
|
||||||
|
|
||||||
|
# Log LAN access addresses if sharing is enabled
|
||||||
|
if lan_sharing:
|
||||||
|
socks_port = config.get("socks5_port", 1080) if config.get("socks5_enabled", True) else None
|
||||||
|
log_lan_access(config.get("listen_port", 8080), socks_port)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
asyncio.run(_run(config))
|
asyncio.run(_run(config))
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
|
|||||||
@@ -12,3 +12,6 @@ brotli>=1.1.0
|
|||||||
|
|
||||||
# Optional: Zstandard decompression (some CDNs now serve `zstd`)
|
# Optional: Zstandard decompression (some CDNs now serve `zstd`)
|
||||||
zstandard>=0.22.0
|
zstandard>=0.22.0
|
||||||
|
|
||||||
|
# Optional: Better network interface detection for LAN sharing
|
||||||
|
netifaces>=0.11.0
|
||||||
|
|||||||
@@ -0,0 +1,191 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Interactive setup wizard for MasterHttpRelayVPN.
|
||||||
|
|
||||||
|
Writes a ready-to-use config.json by prompting only for the values
|
||||||
|
the user really has to choose. Everything else gets a sane default.
|
||||||
|
|
||||||
|
Run:
|
||||||
|
python setup.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import secrets
|
||||||
|
import shutil
|
||||||
|
import string
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
HERE = Path(__file__).resolve().parent
|
||||||
|
CONFIG_PATH = HERE / "config.json"
|
||||||
|
EXAMPLE_PATH = HERE / "config.example.json"
|
||||||
|
|
||||||
|
|
||||||
|
def _c(code: str, text: str) -> str:
|
||||||
|
if os.environ.get("NO_COLOR") or not sys.stdout.isatty():
|
||||||
|
return text
|
||||||
|
return f"\033[{code}m{text}\033[0m"
|
||||||
|
|
||||||
|
|
||||||
|
def bold(t: str) -> str: return _c("1", t)
|
||||||
|
def cyan(t: str) -> str: return _c("36", t)
|
||||||
|
def green(t: str) -> str: return _c("32", t)
|
||||||
|
def yellow(t: str) -> str: return _c("33", t)
|
||||||
|
def red(t: str) -> str: return _c("31", t)
|
||||||
|
def dim(t: str) -> str: return _c("2", t)
|
||||||
|
|
||||||
|
|
||||||
|
def prompt(question: str, default: str | None = None) -> str:
|
||||||
|
suffix = f" [{dim(default)}]" if default else ""
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
raw = input(f"{cyan('?')} {question}{suffix}: ").strip()
|
||||||
|
except EOFError:
|
||||||
|
print()
|
||||||
|
sys.exit(1)
|
||||||
|
if not raw and default is not None:
|
||||||
|
return default
|
||||||
|
if raw:
|
||||||
|
return raw
|
||||||
|
print(red(" value required"))
|
||||||
|
|
||||||
|
|
||||||
|
def prompt_yes_no(question: str, default: bool = True) -> bool:
|
||||||
|
hint = "Y/n" if default else "y/N"
|
||||||
|
while True:
|
||||||
|
raw = input(f"{cyan('?')} {question} [{hint}]: ").strip().lower()
|
||||||
|
if not raw:
|
||||||
|
return default
|
||||||
|
if raw in ("y", "yes"):
|
||||||
|
return True
|
||||||
|
if raw in ("n", "no"):
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def random_auth_key(length: int = 32) -> str:
|
||||||
|
alphabet = string.ascii_letters + string.digits
|
||||||
|
return "".join(secrets.choice(alphabet) for _ in range(length))
|
||||||
|
|
||||||
|
|
||||||
|
def load_base_config() -> dict:
|
||||||
|
if EXAMPLE_PATH.exists():
|
||||||
|
try:
|
||||||
|
with EXAMPLE_PATH.open() as f:
|
||||||
|
return json.load(f)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return {
|
||||||
|
"mode": "apps_script",
|
||||||
|
"google_ip": "216.239.38.120",
|
||||||
|
"front_domain": "www.google.com",
|
||||||
|
"listen_host": "127.0.0.1",
|
||||||
|
"listen_port": 8085,
|
||||||
|
"socks5_enabled": True,
|
||||||
|
"socks5_port": 1080,
|
||||||
|
"log_level": "INFO",
|
||||||
|
"verify_ssl": True,
|
||||||
|
"hosts": {},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def configure_apps_script(cfg: dict) -> dict:
|
||||||
|
print()
|
||||||
|
print(bold("Google Apps Script setup"))
|
||||||
|
print(dim(" 1. Open https://script.google.com -> New project"))
|
||||||
|
print(dim(" 2. Paste apps_script/Code.gs from this repo into the editor"))
|
||||||
|
print(dim(" 3. Set AUTH_KEY in Code.gs to the password below"))
|
||||||
|
print(dim(" 4. Deploy -> New deployment -> Web app"))
|
||||||
|
print(dim(" Execute as: Me | Who has access: Anyone"))
|
||||||
|
print(dim(" 5. Copy the Deployment ID and paste it here"))
|
||||||
|
print()
|
||||||
|
|
||||||
|
ids_raw = prompt(
|
||||||
|
"Deployment ID(s) - comma-separated for load balancing",
|
||||||
|
default=None,
|
||||||
|
)
|
||||||
|
ids = [x.strip() for x in ids_raw.split(",") if x.strip()]
|
||||||
|
if len(ids) == 1:
|
||||||
|
cfg["script_id"] = ids[0]
|
||||||
|
cfg.pop("script_ids", None)
|
||||||
|
else:
|
||||||
|
cfg["script_ids"] = ids
|
||||||
|
cfg.pop("script_id", None)
|
||||||
|
return cfg
|
||||||
|
|
||||||
|
|
||||||
|
def configure_network(cfg: dict) -> dict:
|
||||||
|
print()
|
||||||
|
print(bold("Network settings") + dim(" (press enter to accept defaults)"))
|
||||||
|
cfg["listen_host"] = prompt("Listen host", default=str(cfg.get("listen_host", "127.0.0.1")))
|
||||||
|
|
||||||
|
port = prompt("HTTP proxy port", default=str(cfg.get("listen_port", 8085)))
|
||||||
|
try:
|
||||||
|
cfg["listen_port"] = int(port)
|
||||||
|
except ValueError:
|
||||||
|
cfg["listen_port"] = 8085
|
||||||
|
|
||||||
|
socks5 = prompt_yes_no("Enable SOCKS5 proxy?", default=bool(cfg.get("socks5_enabled", True)))
|
||||||
|
cfg["socks5_enabled"] = socks5
|
||||||
|
if socks5:
|
||||||
|
sport = prompt("SOCKS5 port", default=str(cfg.get("socks5_port", 1080)))
|
||||||
|
try:
|
||||||
|
cfg["socks5_port"] = int(sport)
|
||||||
|
except ValueError:
|
||||||
|
cfg["socks5_port"] = 1080
|
||||||
|
return cfg
|
||||||
|
|
||||||
|
|
||||||
|
def write_config(cfg: dict) -> None:
|
||||||
|
if CONFIG_PATH.exists():
|
||||||
|
backup = CONFIG_PATH.with_suffix(".json.bak")
|
||||||
|
shutil.copy2(CONFIG_PATH, backup)
|
||||||
|
print(yellow(f" existing config.json backed up to {backup.name}"))
|
||||||
|
with CONFIG_PATH.open("w", encoding="utf-8") as f:
|
||||||
|
json.dump(cfg, f, indent=2)
|
||||||
|
f.write("\n")
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
print()
|
||||||
|
print(bold("MasterHttpRelayVPN - setup wizard"))
|
||||||
|
print(dim("Answer a few questions and we'll write config.json for you."))
|
||||||
|
|
||||||
|
if CONFIG_PATH.exists():
|
||||||
|
if not prompt_yes_no("config.json already exists. Overwrite?", default=False):
|
||||||
|
print(dim("Nothing changed."))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
cfg = load_base_config()
|
||||||
|
cfg["mode"] = "apps_script"
|
||||||
|
|
||||||
|
suggested_key = random_auth_key()
|
||||||
|
print()
|
||||||
|
print(bold("Shared password (auth_key)"))
|
||||||
|
print(dim(" Must match AUTH_KEY inside apps_script/Code.gs."))
|
||||||
|
cfg["auth_key"] = prompt("auth_key", default=suggested_key)
|
||||||
|
|
||||||
|
cfg = configure_apps_script(cfg)
|
||||||
|
cfg = configure_network(cfg)
|
||||||
|
|
||||||
|
write_config(cfg)
|
||||||
|
|
||||||
|
print()
|
||||||
|
print(green(f"[OK] wrote {CONFIG_PATH.name}"))
|
||||||
|
print()
|
||||||
|
print(bold("Next step:"))
|
||||||
|
print(f" python main.py")
|
||||||
|
print()
|
||||||
|
print(yellow("Reminder: the AUTH_KEY inside apps_script/Code.gs must match the auth_key"))
|
||||||
|
print(yellow("you just entered - otherwise the relay will return 'unauthorized'."))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
try:
|
||||||
|
sys.exit(main())
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
print()
|
||||||
|
print(dim("Cancelled."))
|
||||||
|
sys.exit(130)
|
||||||
@@ -78,6 +78,35 @@ BATCH_WINDOW_MACRO = 0.050 # 50 ms
|
|||||||
BATCH_MAX = 50
|
BATCH_MAX = 50
|
||||||
|
|
||||||
|
|
||||||
|
# ── Fan-out relay (parallel Apps Script instances) ────────────────────────
|
||||||
|
# How long to ignore a script ID after it fails or is unreasonably slow.
|
||||||
|
SCRIPT_BLACKLIST_TTL = 600.0 # 10 minutes
|
||||||
|
|
||||||
|
|
||||||
|
# ── SNI rotation pool ─────────────────────────────────────────────────────
|
||||||
|
# Google-owned SNIs that share the same edge IPs as www.google.com.
|
||||||
|
# When `front_domain` is a Google property, we rotate through this pool on
|
||||||
|
# each new outbound TLS handshake so DPI systems don't see a constant
|
||||||
|
# "always www.google.com" pattern from the client.
|
||||||
|
FRONT_SNI_POOL_GOOGLE: tuple[str, ...] = (
|
||||||
|
"www.google.com",
|
||||||
|
"mail.google.com",
|
||||||
|
"drive.google.com",
|
||||||
|
"docs.google.com",
|
||||||
|
"calendar.google.com",
|
||||||
|
"maps.google.com",
|
||||||
|
"chat.google.com",
|
||||||
|
"translate.google.com",
|
||||||
|
"play.google.com",
|
||||||
|
"lens.google.com",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ── Per-host stats ────────────────────────────────────────────────────────
|
||||||
|
STATS_LOG_INTERVAL = 300.0 # seconds — how often to log per-host totals
|
||||||
|
STATS_LOG_TOP_N = 10 # how many hosts to include in the log
|
||||||
|
|
||||||
|
|
||||||
# ── Direct Google tunnel allow / exclude ──────────────────────────────────
|
# ── Direct Google tunnel allow / exclude ──────────────────────────────────
|
||||||
# Google web-apps whose real origin must go through the Apps Script relay
|
# Google web-apps whose real origin must go through the Apps Script relay
|
||||||
# because direct SNI tunneling to them does not work reliably behind DPI.
|
# because direct SNI tunneling to them does not work reliably behind DPI.
|
||||||
@@ -101,6 +130,9 @@ GOOGLE_DIRECT_EXACT_EXCLUDE = frozenset({
|
|||||||
"classroom.google.com",
|
"classroom.google.com",
|
||||||
"keep.google.com",
|
"keep.google.com",
|
||||||
"play.google.com",
|
"play.google.com",
|
||||||
|
"translate.google.com",
|
||||||
|
"assistant.google.com",
|
||||||
|
"lens.google.com",
|
||||||
})
|
})
|
||||||
GOOGLE_DIRECT_SUFFIX_EXCLUDE: tuple[str, ...] = (
|
GOOGLE_DIRECT_SUFFIX_EXCLUDE: tuple[str, ...] = (
|
||||||
".meet.google.com",
|
".meet.google.com",
|
||||||
|
|||||||
+362
-30
@@ -14,8 +14,10 @@ import hashlib
|
|||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
|
import socket
|
||||||
import ssl
|
import ssl
|
||||||
import time
|
import time
|
||||||
|
from dataclasses import dataclass
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
import codec
|
import codec
|
||||||
@@ -24,12 +26,16 @@ from constants import (
|
|||||||
BATCH_WINDOW_MACRO,
|
BATCH_WINDOW_MACRO,
|
||||||
BATCH_WINDOW_MICRO,
|
BATCH_WINDOW_MICRO,
|
||||||
CONN_TTL,
|
CONN_TTL,
|
||||||
|
FRONT_SNI_POOL_GOOGLE,
|
||||||
POOL_MAX,
|
POOL_MAX,
|
||||||
POOL_MIN_IDLE,
|
POOL_MIN_IDLE,
|
||||||
RELAY_TIMEOUT,
|
RELAY_TIMEOUT,
|
||||||
|
SCRIPT_BLACKLIST_TTL,
|
||||||
SEMAPHORE_MAX,
|
SEMAPHORE_MAX,
|
||||||
STATEFUL_HEADER_NAMES,
|
STATEFUL_HEADER_NAMES,
|
||||||
STATIC_EXTS,
|
STATIC_EXTS,
|
||||||
|
STATS_LOG_INTERVAL,
|
||||||
|
STATS_LOG_TOP_N,
|
||||||
TLS_CONNECT_TIMEOUT,
|
TLS_CONNECT_TIMEOUT,
|
||||||
WARM_POOL_COUNT,
|
WARM_POOL_COUNT,
|
||||||
)
|
)
|
||||||
@@ -37,12 +43,56 @@ from constants import (
|
|||||||
log = logging.getLogger("Fronter")
|
log = logging.getLogger("Fronter")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class HostStat:
|
||||||
|
"""Per-host traffic accounting — useful for profiling slow / heavy sites."""
|
||||||
|
requests: int = 0
|
||||||
|
cache_hits: int = 0
|
||||||
|
bytes: int = 0
|
||||||
|
total_latency_ns: int = 0
|
||||||
|
errors: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
def _build_sni_pool(front_domain: str, overrides: list | None) -> list[str]:
|
||||||
|
"""Build the list of SNIs to rotate through on new outbound TLS handshakes.
|
||||||
|
|
||||||
|
Priority:
|
||||||
|
1. Explicit `front_domains` list in config (overrides).
|
||||||
|
2. If `front_domain` is a Google property, use FRONT_SNI_POOL_GOOGLE
|
||||||
|
(all share the same Google edge IP, so rotation is invisible to
|
||||||
|
the relay but breaks DPI's "always www.google.com" heuristic).
|
||||||
|
3. Fall back to the single configured `front_domain`.
|
||||||
|
"""
|
||||||
|
if overrides:
|
||||||
|
seen: set[str] = set()
|
||||||
|
out: list[str] = []
|
||||||
|
for item in overrides:
|
||||||
|
host = str(item).strip().lower().rstrip(".")
|
||||||
|
if host and host not in seen:
|
||||||
|
seen.add(host)
|
||||||
|
out.append(host)
|
||||||
|
if out:
|
||||||
|
return out
|
||||||
|
fd = (front_domain or "").lower().rstrip(".")
|
||||||
|
if fd.endswith(".google.com") or fd == "google.com":
|
||||||
|
# Ensure the configured front_domain is first (stable default).
|
||||||
|
pool = [fd] + [h for h in FRONT_SNI_POOL_GOOGLE if h != fd]
|
||||||
|
return pool
|
||||||
|
return [fd] if fd else ["www.google.com"]
|
||||||
|
|
||||||
|
|
||||||
class DomainFronter:
|
class DomainFronter:
|
||||||
_STATIC_EXTS = STATIC_EXTS
|
_STATIC_EXTS = STATIC_EXTS
|
||||||
|
|
||||||
def __init__(self, config: dict):
|
def __init__(self, config: dict):
|
||||||
self.connect_host = config.get("google_ip", "216.239.38.120")
|
self.connect_host = config.get("google_ip", "216.239.38.120")
|
||||||
self.sni_host = config.get("front_domain", "www.google.com")
|
self.sni_host = config.get("front_domain", "www.google.com")
|
||||||
|
# SNI rotation pool — rotated per new outbound TLS connection so
|
||||||
|
# DPI systems can't fingerprint traffic as "always one SNI".
|
||||||
|
self._sni_hosts = _build_sni_pool(
|
||||||
|
self.sni_host, config.get("front_domains"),
|
||||||
|
)
|
||||||
|
self._sni_idx = 0
|
||||||
self.http_host = "script.google.com"
|
self.http_host = "script.google.com"
|
||||||
# Multi-script round-robin for higher throughput
|
# Multi-script round-robin for higher throughput
|
||||||
script = config.get("script_ids") or config.get("script_id")
|
script = config.get("script_ids") or config.get("script_id")
|
||||||
@@ -51,6 +101,23 @@ class DomainFronter:
|
|||||||
self.script_id = self._script_ids[0] # backward compat / logging
|
self.script_id = self._script_ids[0] # backward compat / logging
|
||||||
self._dev_available = False # True if /dev endpoint works (no redirect, ~400ms faster)
|
self._dev_available = False # True if /dev endpoint works (no redirect, ~400ms faster)
|
||||||
|
|
||||||
|
# Fan-out parallel relay: fire N Apps Script instances concurrently,
|
||||||
|
# keep the first successful response, cancel the rest. Script IDs
|
||||||
|
# that fail or time out get blacklisted for SCRIPT_BLACKLIST_TTL so
|
||||||
|
# a single slow container stops poisoning tail latency.
|
||||||
|
try:
|
||||||
|
self._parallel_relay = int(config.get("parallel_relay", 1))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
self._parallel_relay = 1
|
||||||
|
self._parallel_relay = max(1, min(self._parallel_relay,
|
||||||
|
len(self._script_ids)))
|
||||||
|
self._sid_blacklist: dict[str, float] = {}
|
||||||
|
self._blacklist_ttl = SCRIPT_BLACKLIST_TTL
|
||||||
|
|
||||||
|
# Per-host stats (requests, cache hits, bytes, cumulative latency).
|
||||||
|
self._per_site: dict[str, HostStat] = {}
|
||||||
|
self._stats_task: asyncio.Task | None = None
|
||||||
|
|
||||||
self.auth_key = config.get("auth_key", "")
|
self.auth_key = config.get("auth_key", "")
|
||||||
self.verify_ssl = config.get("verify_ssl", True)
|
self.verify_ssl = config.get("verify_ssl", True)
|
||||||
|
|
||||||
@@ -86,13 +153,21 @@ class DomainFronter:
|
|||||||
from h2_transport import H2Transport, H2_AVAILABLE
|
from h2_transport import H2Transport, H2_AVAILABLE
|
||||||
if H2_AVAILABLE:
|
if H2_AVAILABLE:
|
||||||
self._h2 = H2Transport(
|
self._h2 = H2Transport(
|
||||||
self.connect_host, self.sni_host, self.verify_ssl
|
self.connect_host, self.sni_host, self.verify_ssl,
|
||||||
|
sni_hosts=self._sni_hosts,
|
||||||
)
|
)
|
||||||
log.info("HTTP/2 multiplexing available — "
|
log.info("HTTP/2 multiplexing available — "
|
||||||
"all requests will share one connection")
|
"all requests will share one connection")
|
||||||
except ImportError:
|
except ImportError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
if len(self._sni_hosts) > 1:
|
||||||
|
log.info("SNI rotation pool (%d): %s",
|
||||||
|
len(self._sni_hosts), ", ".join(self._sni_hosts))
|
||||||
|
if self._parallel_relay > 1:
|
||||||
|
log.info("Fan-out relay: %d parallel Apps Script instances per request",
|
||||||
|
self._parallel_relay)
|
||||||
|
|
||||||
# Capability log for content encodings.
|
# Capability log for content encodings.
|
||||||
log.info("Response codecs: %s", codec.supported_encodings())
|
log.info("Response codecs: %s", codec.supported_encodings())
|
||||||
|
|
||||||
@@ -108,15 +183,35 @@ class DomainFronter:
|
|||||||
async def _open(self):
|
async def _open(self):
|
||||||
"""Open a TLS connection to the CDN.
|
"""Open a TLS connection to the CDN.
|
||||||
|
|
||||||
The *server_hostname* parameter sets the **TLS SNI** extension.
|
- TCP_NODELAY is set on the underlying socket so small H2/H1 writes
|
||||||
DPI systems see only this value.
|
aren't held back by Nagle's algorithm (up to ~40 ms per batch).
|
||||||
|
- The *server_hostname* parameter sets the **TLS SNI** extension;
|
||||||
|
we rotate across `self._sni_hosts` so DPI can't fingerprint
|
||||||
|
"always www.google.com" from the client side.
|
||||||
"""
|
"""
|
||||||
return await asyncio.open_connection(
|
loop = asyncio.get_event_loop()
|
||||||
self.connect_host,
|
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||||
443,
|
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
||||||
ssl=self._ssl_ctx(),
|
sock.setblocking(False)
|
||||||
server_hostname=self.sni_host,
|
try:
|
||||||
)
|
await loop.sock_connect(sock, (self.connect_host, 443))
|
||||||
|
return await asyncio.open_connection(
|
||||||
|
sock=sock,
|
||||||
|
ssl=self._ssl_ctx(),
|
||||||
|
server_hostname=self._next_sni(),
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
try:
|
||||||
|
sock.close()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
raise
|
||||||
|
|
||||||
|
def _next_sni(self) -> str:
|
||||||
|
"""Round-robin the next SNI from the rotation pool."""
|
||||||
|
sni = self._sni_hosts[self._sni_idx % len(self._sni_hosts)]
|
||||||
|
self._sni_idx += 1
|
||||||
|
return sni
|
||||||
|
|
||||||
async def _acquire(self):
|
async def _acquire(self):
|
||||||
"""Get a healthy TLS connection from pool (TTL-checked) or open new."""
|
"""Get a healthy TLS connection from pool (TTL-checked) or open new."""
|
||||||
@@ -160,11 +255,69 @@ class DomainFronter:
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
def _next_script_id(self) -> str:
|
def _next_script_id(self) -> str:
|
||||||
"""Round-robin across script IDs for load distribution."""
|
"""Round-robin across script IDs for load distribution.
|
||||||
sid = self._script_ids[self._script_idx % len(self._script_ids)]
|
|
||||||
|
Skips script IDs currently in the short-term blacklist (failing
|
||||||
|
or slow) unless *all* are blacklisted, in which case we fall back
|
||||||
|
to plain round-robin so traffic can still flow.
|
||||||
|
"""
|
||||||
|
n = len(self._script_ids)
|
||||||
|
for _ in range(n):
|
||||||
|
sid = self._script_ids[self._script_idx % n]
|
||||||
|
self._script_idx += 1
|
||||||
|
if not self._is_sid_blacklisted(sid):
|
||||||
|
return sid
|
||||||
|
# All blacklisted — clear expired entries and fall back.
|
||||||
|
self._prune_blacklist(force=True)
|
||||||
|
sid = self._script_ids[self._script_idx % n]
|
||||||
self._script_idx += 1
|
self._script_idx += 1
|
||||||
return sid
|
return sid
|
||||||
|
|
||||||
|
def _is_sid_blacklisted(self, sid: str) -> bool:
|
||||||
|
until = self._sid_blacklist.get(sid, 0.0)
|
||||||
|
if until and until > time.time():
|
||||||
|
return True
|
||||||
|
if until:
|
||||||
|
self._sid_blacklist.pop(sid, None)
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _blacklist_sid(self, sid: str, reason: str = "") -> None:
|
||||||
|
"""Blacklist a script ID for SCRIPT_BLACKLIST_TTL seconds."""
|
||||||
|
if len(self._script_ids) <= 1:
|
||||||
|
return # Nothing to fall back to — blacklist would be pointless.
|
||||||
|
self._sid_blacklist[sid] = time.time() + self._blacklist_ttl
|
||||||
|
log.warning("Blacklisted script %s for %ds%s",
|
||||||
|
sid[-8:] if len(sid) > 8 else sid,
|
||||||
|
int(self._blacklist_ttl),
|
||||||
|
f" ({reason})" if reason else "")
|
||||||
|
|
||||||
|
def _prune_blacklist(self, force: bool = False) -> None:
|
||||||
|
now = time.time()
|
||||||
|
for sid, until in list(self._sid_blacklist.items()):
|
||||||
|
if force or until <= now:
|
||||||
|
self._sid_blacklist.pop(sid, None)
|
||||||
|
|
||||||
|
def _pick_fanout_sids(self, key: str | None) -> list[str]:
|
||||||
|
"""Pick up to `parallel_relay` distinct non-blacklisted script IDs.
|
||||||
|
|
||||||
|
The first ID is the stable per-host choice (same as single-shot
|
||||||
|
routing); the rest are filled from the remaining pool. This keeps
|
||||||
|
session-sensitive hosts pinned to one script while still racing
|
||||||
|
extras for lower tail latency.
|
||||||
|
"""
|
||||||
|
if self._parallel_relay <= 1 or len(self._script_ids) <= 1:
|
||||||
|
return [self._script_id_for_key(key)]
|
||||||
|
primary = self._script_id_for_key(key)
|
||||||
|
picked = [primary]
|
||||||
|
others = [s for s in self._script_ids
|
||||||
|
if s != primary and not self._is_sid_blacklisted(s)]
|
||||||
|
# Round-robin-ish selection from `others`
|
||||||
|
for sid in others:
|
||||||
|
if len(picked) >= self._parallel_relay:
|
||||||
|
break
|
||||||
|
picked.append(sid)
|
||||||
|
return picked
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _host_key(url_or_host: str | None) -> str:
|
def _host_key(url_or_host: str | None) -> str:
|
||||||
"""Return a stable routing key for a URL or host string."""
|
"""Return a stable routing key for a URL or host string."""
|
||||||
@@ -174,6 +327,76 @@ class DomainFronter:
|
|||||||
host = parsed.hostname or url_or_host
|
host = parsed.hostname or url_or_host
|
||||||
return host.lower().rstrip(".")
|
return host.lower().rstrip(".")
|
||||||
|
|
||||||
|
# ── Per-host stats ────────────────────────────────────────────
|
||||||
|
|
||||||
|
def _record_site(self, url: str, bytes_: int, latency_ns: int,
|
||||||
|
errored: bool) -> None:
|
||||||
|
host = self._host_key(url)
|
||||||
|
if not host:
|
||||||
|
return
|
||||||
|
stat = self._per_site.get(host)
|
||||||
|
if stat is None:
|
||||||
|
stat = HostStat()
|
||||||
|
self._per_site[host] = stat
|
||||||
|
stat.requests += 1
|
||||||
|
stat.bytes += max(0, int(bytes_))
|
||||||
|
stat.total_latency_ns += max(0, int(latency_ns))
|
||||||
|
if errored:
|
||||||
|
stat.errors += 1
|
||||||
|
|
||||||
|
def stats_snapshot(self) -> dict:
|
||||||
|
"""Return a point-in-time snapshot of traffic + script health."""
|
||||||
|
per_site = []
|
||||||
|
for host, s in self._per_site.items():
|
||||||
|
avg_ms = (s.total_latency_ns / s.requests / 1e6) if s.requests else 0.0
|
||||||
|
per_site.append({
|
||||||
|
"host": host,
|
||||||
|
"requests": s.requests,
|
||||||
|
"errors": s.errors,
|
||||||
|
"bytes": s.bytes,
|
||||||
|
"avg_ms": round(avg_ms, 1),
|
||||||
|
})
|
||||||
|
per_site.sort(key=lambda x: x["bytes"], reverse=True)
|
||||||
|
now = time.time()
|
||||||
|
blacklisted = [
|
||||||
|
{"sid": sid[-12:] if len(sid) > 12 else sid,
|
||||||
|
"expires_in_s": int(max(0, until - now))}
|
||||||
|
for sid, until in self._sid_blacklist.items() if until > now
|
||||||
|
]
|
||||||
|
return {
|
||||||
|
"per_site": per_site,
|
||||||
|
"blacklisted_scripts": blacklisted,
|
||||||
|
"sni_rotation": list(self._sni_hosts),
|
||||||
|
"parallel_relay": self._parallel_relay,
|
||||||
|
}
|
||||||
|
|
||||||
|
async def _stats_logger(self):
|
||||||
|
"""Periodically log top hosts by bytes. DEBUG-level, low overhead."""
|
||||||
|
interval = STATS_LOG_INTERVAL
|
||||||
|
top_n = STATS_LOG_TOP_N
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
await asyncio.sleep(interval)
|
||||||
|
if not log.isEnabledFor(logging.DEBUG) or not self._per_site:
|
||||||
|
continue
|
||||||
|
snap = self.stats_snapshot()
|
||||||
|
top = snap["per_site"][:top_n]
|
||||||
|
log.debug("── Per-host stats (top %d by bytes) ──", len(top))
|
||||||
|
for row in top:
|
||||||
|
log.debug(
|
||||||
|
" %-40s %5d req %2d err %8d KB avg %7.1f ms",
|
||||||
|
row["host"][:40], row["requests"], row["errors"],
|
||||||
|
row["bytes"] // 1024, row["avg_ms"],
|
||||||
|
)
|
||||||
|
if snap["blacklisted_scripts"]:
|
||||||
|
log.debug(" blacklisted scripts: %s",
|
||||||
|
", ".join(f"{b['sid']} ({b['expires_in_s']}s)"
|
||||||
|
for b in snap["blacklisted_scripts"]))
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
break
|
||||||
|
except Exception as e:
|
||||||
|
log.debug("Stats logger error: %s", e)
|
||||||
|
|
||||||
def _script_id_for_key(self, key: str | None = None) -> str:
|
def _script_id_for_key(self, key: str | None = None) -> str:
|
||||||
"""Pick a stable Apps Script ID for a host or fallback to round-robin.
|
"""Pick a stable Apps Script ID for a host or fallback to round-robin.
|
||||||
|
|
||||||
@@ -181,20 +404,31 @@ class DomainFronter:
|
|||||||
host reduces IP/session churn for sites that are sensitive to endpoint
|
host reduces IP/session churn for sites that are sensitive to endpoint
|
||||||
changes. If no key is available, we keep the older round-robin fallback
|
changes. If no key is available, we keep the older round-robin fallback
|
||||||
so warmup/keepalive traffic still distributes normally.
|
so warmup/keepalive traffic still distributes normally.
|
||||||
|
|
||||||
|
Blacklisted IDs are skipped by probing forward in the list until a
|
||||||
|
healthy one is found; if none, the stable pick is returned anyway.
|
||||||
"""
|
"""
|
||||||
if len(self._script_ids) == 1:
|
if len(self._script_ids) == 1:
|
||||||
return self._script_ids[0]
|
return self._script_ids[0]
|
||||||
if not key:
|
if not key:
|
||||||
return self._next_script_id()
|
return self._next_script_id()
|
||||||
digest = hashlib.sha1(key.encode("utf-8")).digest()
|
digest = hashlib.sha1(key.encode("utf-8")).digest()
|
||||||
idx = int.from_bytes(digest[:4], "big") % len(self._script_ids)
|
base = int.from_bytes(digest[:4], "big") % len(self._script_ids)
|
||||||
return self._script_ids[idx]
|
n = len(self._script_ids)
|
||||||
|
for offset in range(n):
|
||||||
|
sid = self._script_ids[(base + offset) % n]
|
||||||
|
if not self._is_sid_blacklisted(sid):
|
||||||
|
return sid
|
||||||
|
return self._script_ids[base]
|
||||||
|
|
||||||
def _exec_path(self, url_or_host: str | None = None) -> str:
|
def _exec_path(self, url_or_host: str | None = None) -> str:
|
||||||
"""Get the Apps Script endpoint path (/dev or /exec)."""
|
"""Get the Apps Script endpoint path (/dev or /exec)."""
|
||||||
sid = self._script_id_for_key(self._host_key(url_or_host))
|
sid = self._script_id_for_key(self._host_key(url_or_host))
|
||||||
return f"/macros/s/{sid}/{'dev' if self._dev_available else 'exec'}"
|
return self._exec_path_for_sid(sid)
|
||||||
|
|
||||||
|
def _exec_path_for_sid(self, sid: str) -> str:
|
||||||
|
"""Build the /macros/s/<sid>/(dev|exec) path for a specific script ID."""
|
||||||
|
return f"/macros/s/{sid}/{'dev' if self._dev_available else 'exec'}"
|
||||||
async def _flush_pool(self):
|
async def _flush_pool(self):
|
||||||
"""Close all pooled connections (they may be stale after errors)."""
|
"""Close all pooled connections (they may be stale after errors)."""
|
||||||
async with self._pool_lock:
|
async with self._pool_lock:
|
||||||
@@ -271,6 +505,9 @@ class DomainFronter:
|
|||||||
# Start continuous pool maintenance
|
# Start continuous pool maintenance
|
||||||
if self._maintenance_task is None:
|
if self._maintenance_task is None:
|
||||||
self._maintenance_task = self._spawn(self._pool_maintenance())
|
self._maintenance_task = self._spawn(self._pool_maintenance())
|
||||||
|
# Periodic per-host stats logger (opt-in via log level)
|
||||||
|
if self._stats_task is None:
|
||||||
|
self._stats_task = self._spawn(self._stats_logger())
|
||||||
# Start H2 connection (runs alongside H1 pool)
|
# Start H2 connection (runs alongside H1 pool)
|
||||||
if self._h2:
|
if self._h2:
|
||||||
self._spawn(self._h2_connect_and_warm())
|
self._spawn(self._h2_connect_and_warm())
|
||||||
@@ -424,24 +661,37 @@ class DomainFronter:
|
|||||||
|
|
||||||
payload = self._build_payload(method, url, headers, body)
|
payload = self._build_payload(method, url, headers, body)
|
||||||
|
|
||||||
# Stateful/browser-navigation requests should preserve exact ordering
|
t0 = time.perf_counter()
|
||||||
# and header context; batching/coalescing is reserved for static fetches.
|
errored = False
|
||||||
if self._is_stateful_request(method, url, headers, body):
|
result: bytes = b""
|
||||||
return await self._relay_with_retry(payload)
|
try:
|
||||||
|
# Stateful/browser-navigation requests should preserve exact ordering
|
||||||
|
# and header context; batching/coalescing is reserved for static fetches.
|
||||||
|
if self._is_stateful_request(method, url, headers, body):
|
||||||
|
result = await self._relay_with_retry(payload)
|
||||||
|
return result
|
||||||
|
|
||||||
# Coalesce concurrent GETs for the same URL.
|
# Coalesce concurrent GETs for the same URL.
|
||||||
# CRITICAL: do NOT coalesce when a Range header is present —
|
# CRITICAL: do NOT coalesce when a Range header is present —
|
||||||
# parallel range downloads MUST each hit the server independently.
|
# parallel range downloads MUST each hit the server independently.
|
||||||
has_range = False
|
has_range = False
|
||||||
if headers:
|
if headers:
|
||||||
for k in headers:
|
for k in headers:
|
||||||
if k.lower() == "range":
|
if k.lower() == "range":
|
||||||
has_range = True
|
has_range = True
|
||||||
break
|
break
|
||||||
if method == "GET" and not body and not has_range:
|
if method == "GET" and not body and not has_range:
|
||||||
return await self._coalesced_submit(url, payload)
|
result = await self._coalesced_submit(url, payload)
|
||||||
|
return result
|
||||||
|
|
||||||
return await self._batch_submit(payload)
|
result = await self._batch_submit(payload)
|
||||||
|
return result
|
||||||
|
except Exception:
|
||||||
|
errored = True
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
latency_ns = int((time.perf_counter() - t0) * 1e9)
|
||||||
|
self._record_site(url, len(result), latency_ns, errored)
|
||||||
|
|
||||||
async def _coalesced_submit(self, url: str, payload: dict) -> bytes:
|
async def _coalesced_submit(self, url: str, payload: dict) -> bytes:
|
||||||
"""Dedup concurrent requests for the same URL (no Range header).
|
"""Dedup concurrent requests for the same URL (no Range header).
|
||||||
@@ -789,6 +1039,20 @@ class DomainFronter:
|
|||||||
|
|
||||||
async def _relay_with_retry(self, payload: dict) -> bytes:
|
async def _relay_with_retry(self, payload: dict) -> bytes:
|
||||||
"""Single relay with one retry on failure. Uses H2 if available."""
|
"""Single relay with one retry on failure. Uses H2 if available."""
|
||||||
|
# Fan-out: race N Apps Script instances when enabled and H2 is up.
|
||||||
|
# Cuts tail latency when one container is slow/cold. Only kicks in
|
||||||
|
# if multiple script IDs are configured and the H2 transport is live.
|
||||||
|
if (self._parallel_relay > 1
|
||||||
|
and len(self._script_ids) > 1
|
||||||
|
and self._h2 and self._h2.is_connected):
|
||||||
|
try:
|
||||||
|
return await asyncio.wait_for(
|
||||||
|
self._relay_fanout(payload), timeout=RELAY_TIMEOUT,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
log.debug("Fan-out relay failed (%s), falling back", e)
|
||||||
|
# fall through to single-path logic below
|
||||||
|
|
||||||
# Try HTTP/2 first — much faster (multiplexed, no pool checkout)
|
# Try HTTP/2 first — much faster (multiplexed, no pool checkout)
|
||||||
if self._h2 and self._h2.is_connected:
|
if self._h2 and self._h2.is_connected:
|
||||||
for attempt in range(2):
|
for attempt in range(2):
|
||||||
@@ -822,6 +1086,53 @@ class DomainFronter:
|
|||||||
else:
|
else:
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
async def _relay_fanout(self, payload: dict) -> bytes:
|
||||||
|
"""Fire the same relay against N distinct script IDs in parallel.
|
||||||
|
|
||||||
|
Returns the first successful response; cancels the rest as soon as
|
||||||
|
one finishes. Any script that raises or loses the race AND later
|
||||||
|
fails individually is blacklisted for SCRIPT_BLACKLIST_TTL.
|
||||||
|
"""
|
||||||
|
host_key = self._host_key(payload.get("u"))
|
||||||
|
sids = self._pick_fanout_sids(host_key)
|
||||||
|
if len(sids) <= 1:
|
||||||
|
# Nothing to race against (e.g. all others blacklisted)
|
||||||
|
return await self._relay_single_h2_with_sid(payload, sids[0])
|
||||||
|
|
||||||
|
tasks = {
|
||||||
|
asyncio.create_task(
|
||||||
|
self._relay_single_h2_with_sid(payload, sid)
|
||||||
|
): sid
|
||||||
|
for sid in sids
|
||||||
|
}
|
||||||
|
winner_result: bytes | None = None
|
||||||
|
winner_exc: BaseException | None = None
|
||||||
|
pending = set(tasks.keys())
|
||||||
|
try:
|
||||||
|
while pending:
|
||||||
|
done, pending = await asyncio.wait(
|
||||||
|
pending, return_when=asyncio.FIRST_COMPLETED,
|
||||||
|
)
|
||||||
|
for t in done:
|
||||||
|
sid = tasks[t]
|
||||||
|
exc = t.exception()
|
||||||
|
if exc is None:
|
||||||
|
winner_result = t.result()
|
||||||
|
return winner_result
|
||||||
|
# This racer failed — blacklist and keep waiting for others
|
||||||
|
self._blacklist_sid(sid, reason=type(exc).__name__)
|
||||||
|
winner_exc = exc
|
||||||
|
# All racers failed
|
||||||
|
if winner_exc is not None:
|
||||||
|
raise winner_exc
|
||||||
|
raise RuntimeError("fan-out relay: all racers failed")
|
||||||
|
finally:
|
||||||
|
for t in pending:
|
||||||
|
t.cancel()
|
||||||
|
# Drain cancelled tasks so they don't log warnings
|
||||||
|
if pending:
|
||||||
|
await asyncio.gather(*pending, return_exceptions=True)
|
||||||
|
|
||||||
async def _relay_single_h2(self, payload: dict) -> bytes:
|
async def _relay_single_h2(self, payload: dict) -> bytes:
|
||||||
"""Execute a relay through HTTP/2 multiplexing.
|
"""Execute a relay through HTTP/2 multiplexing.
|
||||||
|
|
||||||
@@ -842,6 +1153,27 @@ class DomainFronter:
|
|||||||
|
|
||||||
return self._parse_relay_response(body)
|
return self._parse_relay_response(body)
|
||||||
|
|
||||||
|
async def _relay_single_h2_with_sid(self, payload: dict,
|
||||||
|
sid: str) -> bytes:
|
||||||
|
"""Execute an H2 relay pinned to a specific Apps Script deployment.
|
||||||
|
|
||||||
|
Used by `_relay_fanout` to race multiple script IDs in parallel.
|
||||||
|
Mirrors `_relay_single_h2` but ignores the stable-hash routing.
|
||||||
|
"""
|
||||||
|
full_payload = dict(payload)
|
||||||
|
full_payload["k"] = self.auth_key
|
||||||
|
json_body = json.dumps(full_payload).encode()
|
||||||
|
|
||||||
|
path = self._exec_path_for_sid(sid)
|
||||||
|
|
||||||
|
status, headers, body = await self._h2.request(
|
||||||
|
method="POST", path=path, host=self.http_host,
|
||||||
|
headers={"content-type": "application/json"},
|
||||||
|
body=json_body,
|
||||||
|
)
|
||||||
|
|
||||||
|
return self._parse_relay_response(body)
|
||||||
|
|
||||||
async def _relay_single(self, payload: dict) -> bytes:
|
async def _relay_single(self, payload: dict) -> bytes:
|
||||||
"""Execute a single relay POST → redirect → parse."""
|
"""Execute a single relay POST → redirect → parse."""
|
||||||
# Add auth key
|
# Add auth key
|
||||||
|
|||||||
+14
-3
@@ -62,10 +62,15 @@ class H2Transport:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, connect_host: str, sni_host: str,
|
def __init__(self, connect_host: str, sni_host: str,
|
||||||
verify_ssl: bool = True):
|
verify_ssl: bool = True,
|
||||||
|
sni_hosts: list[str] | None = None):
|
||||||
self.connect_host = connect_host
|
self.connect_host = connect_host
|
||||||
self.sni_host = sni_host
|
self.sni_host = sni_host
|
||||||
self.verify_ssl = verify_ssl
|
self.verify_ssl = verify_ssl
|
||||||
|
# Optional SNI rotation pool — picked round-robin on each new connect.
|
||||||
|
# Falls back to the single sni_host if no pool is given.
|
||||||
|
self._sni_hosts: list[str] = [h for h in (sni_hosts or []) if h] or [sni_host]
|
||||||
|
self._sni_idx: int = 0
|
||||||
|
|
||||||
self._reader: asyncio.StreamReader | None = None
|
self._reader: asyncio.StreamReader | None = None
|
||||||
self._writer: asyncio.StreamWriter | None = None
|
self._writer: asyncio.StreamWriter | None = None
|
||||||
@@ -107,6 +112,12 @@ class H2Transport:
|
|||||||
ctx.check_hostname = False
|
ctx.check_hostname = False
|
||||||
ctx.verify_mode = ssl.CERT_NONE
|
ctx.verify_mode = ssl.CERT_NONE
|
||||||
|
|
||||||
|
# Pick next SNI from the rotation pool so repeated reconnects
|
||||||
|
# don't fingerprint as "always www.google.com".
|
||||||
|
sni = self._sni_hosts[self._sni_idx % len(self._sni_hosts)]
|
||||||
|
self._sni_idx += 1
|
||||||
|
self.sni_host = sni # kept for backward-compat logging
|
||||||
|
|
||||||
# Create raw TCP socket with TCP_NODELAY BEFORE TLS handshake.
|
# Create raw TCP socket with TCP_NODELAY BEFORE TLS handshake.
|
||||||
# Nagle's algorithm can delay small writes (H2 frames) by up to 200ms
|
# Nagle's algorithm can delay small writes (H2 frames) by up to 200ms
|
||||||
# waiting to coalesce — TCP_NODELAY forces immediate send.
|
# waiting to coalesce — TCP_NODELAY forces immediate send.
|
||||||
@@ -124,7 +135,7 @@ class H2Transport:
|
|||||||
self._reader, self._writer = await asyncio.wait_for(
|
self._reader, self._writer = await asyncio.wait_for(
|
||||||
asyncio.open_connection(
|
asyncio.open_connection(
|
||||||
ssl=ctx,
|
ssl=ctx,
|
||||||
server_hostname=self.sni_host,
|
server_hostname=sni,
|
||||||
sock=raw,
|
sock=raw,
|
||||||
),
|
),
|
||||||
timeout=15,
|
timeout=15,
|
||||||
@@ -165,7 +176,7 @@ class H2Transport:
|
|||||||
self._connected = True
|
self._connected = True
|
||||||
self._read_task = asyncio.create_task(self._reader_loop())
|
self._read_task = asyncio.create_task(self._reader_loop())
|
||||||
log.info("H2 connected → %s (SNI=%s, TCP_NODELAY=on)",
|
log.info("H2 connected → %s (SNI=%s, TCP_NODELAY=on)",
|
||||||
self.connect_host, self.sni_host)
|
self.connect_host, sni)
|
||||||
|
|
||||||
async def reconnect(self):
|
async def reconnect(self):
|
||||||
"""Close current connection and re-establish."""
|
"""Close current connection and re-establish."""
|
||||||
|
|||||||
@@ -0,0 +1,152 @@
|
|||||||
|
"""
|
||||||
|
LAN utilities for detecting network interfaces and IP addresses.
|
||||||
|
|
||||||
|
Provides functionality to enumerate local network interfaces and their
|
||||||
|
associated IP addresses for LAN proxy sharing.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import ipaddress
|
||||||
|
import logging
|
||||||
|
import socket
|
||||||
|
from typing import Dict, List, Optional
|
||||||
|
|
||||||
|
log = logging.getLogger("LAN")
|
||||||
|
|
||||||
|
|
||||||
|
def get_network_interfaces() -> Dict[str, List[str]]:
|
||||||
|
"""
|
||||||
|
Get all network interfaces and their associated IP addresses.
|
||||||
|
|
||||||
|
Returns a dictionary mapping interface names to lists of IP addresses
|
||||||
|
(both IPv4 and IPv6). Only includes interfaces with valid IP addresses
|
||||||
|
that are not loopback.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict[str, List[str]]: Interface name -> list of IP addresses
|
||||||
|
"""
|
||||||
|
interfaces = {}
|
||||||
|
|
||||||
|
try:
|
||||||
|
import netifaces
|
||||||
|
for iface in netifaces.interfaces():
|
||||||
|
addrs = netifaces.ifaddresses(iface)
|
||||||
|
ips = []
|
||||||
|
# IPv4 addresses
|
||||||
|
if netifaces.AF_INET in addrs:
|
||||||
|
for addr in addrs[netifaces.AF_INET]:
|
||||||
|
ip = addr.get('addr')
|
||||||
|
if ip and not ip.startswith('127.'):
|
||||||
|
ips.append(ip)
|
||||||
|
# IPv6 addresses (without scope)
|
||||||
|
if netifaces.AF_INET6 in addrs:
|
||||||
|
for addr in addrs[netifaces.AF_INET6]:
|
||||||
|
ip = addr.get('addr')
|
||||||
|
if ip and not ip.startswith('::1') and not '%' in ip:
|
||||||
|
# Remove scope if present
|
||||||
|
ips.append(ip.split('%')[0])
|
||||||
|
if ips:
|
||||||
|
interfaces[iface] = ips
|
||||||
|
except ImportError:
|
||||||
|
# Fallback to socket method for basic detection
|
||||||
|
log.debug("netifaces not available, using socket fallback")
|
||||||
|
interfaces = _get_interfaces_socket_fallback()
|
||||||
|
|
||||||
|
return interfaces
|
||||||
|
|
||||||
|
|
||||||
|
def _get_interfaces_socket_fallback() -> Dict[str, List[str]]:
|
||||||
|
"""
|
||||||
|
Fallback method to get network interfaces using socket.
|
||||||
|
|
||||||
|
This is less comprehensive than netifaces but works without extra dependencies.
|
||||||
|
"""
|
||||||
|
interfaces = {}
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Get hostname and try to resolve to IPs
|
||||||
|
hostname = socket.gethostname()
|
||||||
|
try:
|
||||||
|
# Get IPv4 addresses
|
||||||
|
ipv4_info = socket.getaddrinfo(hostname, None, socket.AF_INET)
|
||||||
|
ipv4_addrs = [info[4][0] for info in ipv4_info if not info[4][0].startswith('127.')]
|
||||||
|
if ipv4_addrs:
|
||||||
|
interfaces['primary'] = list(set(ipv4_addrs)) # Remove duplicates
|
||||||
|
except socket.gaierror:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Get IPv6 addresses
|
||||||
|
ipv6_info = socket.getaddrinfo(hostname, None, socket.AF_INET6)
|
||||||
|
ipv6_addrs = []
|
||||||
|
for info in ipv6_info:
|
||||||
|
ip = info[4][0]
|
||||||
|
if not ip.startswith('::1') and not '%' in ip:
|
||||||
|
ipv6_addrs.append(ip.split('%')[0])
|
||||||
|
if ipv6_addrs:
|
||||||
|
interfaces['primary_ipv6'] = list(set(ipv6_addrs))
|
||||||
|
except socket.gaierror:
|
||||||
|
pass
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
log.debug("Socket fallback failed: %s", e)
|
||||||
|
|
||||||
|
return interfaces
|
||||||
|
|
||||||
|
|
||||||
|
def get_lan_ips(port: int = 8085) -> List[str]:
|
||||||
|
"""
|
||||||
|
Get list of LAN-accessible proxy addresses.
|
||||||
|
|
||||||
|
Returns a list of IP:port combinations that can be used to access
|
||||||
|
the proxy from other devices on the local network.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
port: The port the proxy is listening on
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List[str]: List of "IP:port" strings for LAN access
|
||||||
|
"""
|
||||||
|
interfaces = get_network_interfaces()
|
||||||
|
lan_addresses = []
|
||||||
|
|
||||||
|
for iface_ips in interfaces.values():
|
||||||
|
for ip in iface_ips:
|
||||||
|
try:
|
||||||
|
# Validate IP and check if it's a private address
|
||||||
|
addr = ipaddress.ip_address(ip)
|
||||||
|
if addr.is_private or addr.is_link_local:
|
||||||
|
lan_addresses.append(f"{ip}:{port}")
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Remove duplicates while preserving order
|
||||||
|
seen = set()
|
||||||
|
unique_addresses = []
|
||||||
|
for addr in lan_addresses:
|
||||||
|
if addr not in seen:
|
||||||
|
seen.add(addr)
|
||||||
|
unique_addresses.append(addr)
|
||||||
|
|
||||||
|
return unique_addresses
|
||||||
|
|
||||||
|
|
||||||
|
def log_lan_access(port: int = 8085, socks_port: Optional[int] = None):
|
||||||
|
"""
|
||||||
|
Log the LAN-accessible proxy addresses for user convenience.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
port: HTTP proxy port
|
||||||
|
socks_port: Optional SOCKS5 proxy port
|
||||||
|
"""
|
||||||
|
lan_http = get_lan_ips(port)
|
||||||
|
if lan_http:
|
||||||
|
log.info("LAN HTTP proxy : %s", ", ".join(lan_http))
|
||||||
|
else:
|
||||||
|
log.warning("No LAN IP addresses detected for HTTP proxy")
|
||||||
|
|
||||||
|
if socks_port:
|
||||||
|
lan_socks = get_lan_ips(socks_port)
|
||||||
|
if lan_socks:
|
||||||
|
log.info("LAN SOCKS5 proxy : %s", ", ".join(lan_socks))
|
||||||
|
else:
|
||||||
|
log.warning("No LAN IP addresses detected for SOCKS5 proxy")
|
||||||
+55
-12
@@ -264,29 +264,68 @@ class ProxyServer:
|
|||||||
def _log_response_summary(self, url: str, response: bytes):
|
def _log_response_summary(self, url: str, response: bytes):
|
||||||
status, headers, body = self.fronter._split_raw_response(response)
|
status, headers, body = self.fronter._split_raw_response(response)
|
||||||
host = (urlparse(url).hostname or "").lower()
|
host = (urlparse(url).hostname or "").lower()
|
||||||
|
|
||||||
if status >= 300 or self._should_trace_host(host):
|
if status >= 300 or self._should_trace_host(host):
|
||||||
location = headers.get("location", "")
|
location = headers.get("location", "") or "-"
|
||||||
server = headers.get("server", "")
|
server = headers.get("server", "") or "-"
|
||||||
cf_ray = headers.get("cf-ray", "")
|
cf_ray = headers.get("cf-ray", "") or "-"
|
||||||
content_type = headers.get("content-type", "")
|
content_type = headers.get("content-type", "") or "-"
|
||||||
body_len = len(body)
|
body_len = len(body)
|
||||||
|
|
||||||
body_hint = "-"
|
body_hint = "-"
|
||||||
if "text/html" in content_type.lower() and body:
|
rate_limited = False
|
||||||
sample = body[:800].decode(errors="replace").lower()
|
|
||||||
|
# Handle text-like responses (HTML, plain text, JSON…)
|
||||||
|
if ("text" in content_type.lower() or "json" in content_type.lower()) and body:
|
||||||
|
sample = body[:1200].decode(errors="replace").lower()
|
||||||
|
|
||||||
|
# --- Structured HTML title extraction ---
|
||||||
if "<title>" in sample and "</title>" in sample:
|
if "<title>" in sample and "</title>" in sample:
|
||||||
title = sample.split("<title>", 1)[1].split("</title>", 1)[0]
|
title = sample.split("<title>", 1)[1].split("</title>", 1)[0]
|
||||||
body_hint = title[:120]
|
body_hint = title.strip()[:120] or "-"
|
||||||
|
|
||||||
|
# --- Known content patterns ---
|
||||||
elif "captcha" in sample:
|
elif "captcha" in sample:
|
||||||
body_hint = "captcha"
|
body_hint = "captcha"
|
||||||
elif "turnstile" in sample:
|
elif "turnstile" in sample:
|
||||||
body_hint = "turnstile"
|
body_hint = "turnstile"
|
||||||
elif "loading" in sample:
|
elif "loading" in sample:
|
||||||
body_hint = "loading"
|
body_hint = "loading"
|
||||||
log.info(
|
|
||||||
"RESP ← %s status=%s type=%s len=%s server=%s location=%s cf-ray=%s hint=%s",
|
# --- Rate-limit / quota markers ---
|
||||||
host or url[:60], status, content_type or "-", body_len,
|
rate_limit_markers = (
|
||||||
server or "-", location or "-", cf_ray or "-", body_hint,
|
"too many",
|
||||||
|
"rate limit",
|
||||||
|
"quota",
|
||||||
|
"quota exceeded",
|
||||||
|
"request limit",
|
||||||
|
"دفعات زیاد",
|
||||||
|
"بیش از حد",
|
||||||
|
"سرویس در طول یک روز",
|
||||||
|
)
|
||||||
|
|
||||||
|
if any(m in sample for m in rate_limit_markers):
|
||||||
|
rate_limited = True
|
||||||
|
body_hint = "quota_exceeded"
|
||||||
|
|
||||||
|
log_msg = (
|
||||||
|
"RESP ← %s status=%s type=%s len=%s server=%s location=%s cf-ray=%s hint=%s"
|
||||||
)
|
)
|
||||||
|
log_args = (
|
||||||
|
host or url[:60],
|
||||||
|
status,
|
||||||
|
content_type,
|
||||||
|
body_len,
|
||||||
|
server,
|
||||||
|
location,
|
||||||
|
cf_ray,
|
||||||
|
body_hint,
|
||||||
|
)
|
||||||
|
|
||||||
|
if rate_limited:
|
||||||
|
log.warning("RATE LIMIT detected! " + log_msg, *log_args)
|
||||||
|
else:
|
||||||
|
log.info(log_msg, *log_args)
|
||||||
|
|
||||||
async def start(self):
|
async def start(self):
|
||||||
http_srv = await asyncio.start_server(self._on_client, self.host, self.port)
|
http_srv = await asyncio.start_server(self._on_client, self.host, self.port)
|
||||||
@@ -951,7 +990,11 @@ class ProxyServer:
|
|||||||
if b":" in raw_line:
|
if b":" in raw_line:
|
||||||
k, v = raw_line.decode(errors="replace").split(":", 1)
|
k, v = raw_line.decode(errors="replace").split(":", 1)
|
||||||
headers[k.strip()] = v.strip()
|
headers[k.strip()] = v.strip()
|
||||||
|
|
||||||
|
# Shortening the length of X API URLs to prevent relay errors.
|
||||||
|
if host == "x.com" and re.match(r"/i/api/graphql/[^/]+/[^?]+\?variables=", path):
|
||||||
|
path = path.split("&")[0]
|
||||||
|
|
||||||
# MITM traffic arrives as origin-form paths; SOCKS/plain HTTP can
|
# MITM traffic arrives as origin-form paths; SOCKS/plain HTTP can
|
||||||
# also send absolute-form requests. Normalize both to full URLs.
|
# also send absolute-form requests. Normalize both to full URLs.
|
||||||
if path.startswith("http://") or path.startswith("https://"):
|
if path.startswith("http://") or path.startswith("https://"):
|
||||||
|
|||||||
@@ -0,0 +1,72 @@
|
|||||||
|
@echo off
|
||||||
|
setlocal enabledelayedexpansion
|
||||||
|
cd /d "%~dp0"
|
||||||
|
|
||||||
|
REM -------- MasterHttpRelayVPN one-click launcher (Windows) --------
|
||||||
|
REM Creates a local virtualenv, installs deps, runs the setup wizard
|
||||||
|
REM if needed, then starts the proxy.
|
||||||
|
|
||||||
|
set "VENV_DIR=.venv"
|
||||||
|
set "PY="
|
||||||
|
|
||||||
|
where py >nul 2>&1
|
||||||
|
if %errorlevel%==0 (
|
||||||
|
set "PY=py -3"
|
||||||
|
) else (
|
||||||
|
where python >nul 2>&1
|
||||||
|
if %errorlevel%==0 (
|
||||||
|
set "PY=python"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if "%PY%"=="" (
|
||||||
|
echo [X] Python 3.10+ was not found on PATH.
|
||||||
|
echo Install from https://www.python.org/downloads/ and re-run this script.
|
||||||
|
pause
|
||||||
|
exit /b 1
|
||||||
|
)
|
||||||
|
|
||||||
|
if not exist "%VENV_DIR%\Scripts\python.exe" (
|
||||||
|
echo [*] Creating virtual environment in %VENV_DIR% ...
|
||||||
|
%PY% -m venv "%VENV_DIR%"
|
||||||
|
if errorlevel 1 (
|
||||||
|
echo [X] Failed to create virtualenv.
|
||||||
|
pause
|
||||||
|
exit /b 1
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
set "VPY=%VENV_DIR%\Scripts\python.exe"
|
||||||
|
|
||||||
|
echo [*] Installing dependencies ...
|
||||||
|
"%VPY%" -m pip install --disable-pip-version-check -q --upgrade pip >nul
|
||||||
|
"%VPY%" -m pip install --disable-pip-version-check -q -r requirements.txt
|
||||||
|
if errorlevel 1 (
|
||||||
|
echo [!] PyPI install failed. Retrying via runflare mirror ...
|
||||||
|
"%VPY%" -m pip install --disable-pip-version-check -q -r requirements.txt ^
|
||||||
|
-i https://mirror-pypi.runflare.com/simple/ ^
|
||||||
|
--trusted-host mirror-pypi.runflare.com
|
||||||
|
if errorlevel 1 (
|
||||||
|
echo [X] Could not install dependencies.
|
||||||
|
pause
|
||||||
|
exit /b 1
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not exist "config.json" (
|
||||||
|
echo [*] No config.json found — launching setup wizard ...
|
||||||
|
"%VPY%" setup.py
|
||||||
|
if errorlevel 1 (
|
||||||
|
echo [X] Setup cancelled.
|
||||||
|
pause
|
||||||
|
exit /b 1
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
echo.
|
||||||
|
echo [*] Starting MasterHttpRelayVPN ...
|
||||||
|
echo.
|
||||||
|
"%VPY%" main.py %*
|
||||||
|
set "RC=%errorlevel%"
|
||||||
|
if not "%RC%"=="0" pause
|
||||||
|
exit /b %RC%
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# MasterHttpRelayVPN one-click launcher (Linux / macOS)
|
||||||
|
# Creates a local virtualenv, installs deps, runs the setup wizard
|
||||||
|
# if needed, then starts the proxy.
|
||||||
|
|
||||||
|
set -e
|
||||||
|
cd "$(dirname "$0")"
|
||||||
|
|
||||||
|
VENV_DIR=".venv"
|
||||||
|
|
||||||
|
find_python() {
|
||||||
|
for cmd in python3.12 python3.11 python3.10 python3 python; do
|
||||||
|
if command -v "$cmd" >/dev/null 2>&1; then
|
||||||
|
ver=$("$cmd" -c 'import sys;print("%d.%d"%sys.version_info[:2])' 2>/dev/null || echo "0.0")
|
||||||
|
major=${ver%.*}; minor=${ver#*.}
|
||||||
|
if [ "$major" -ge 3 ] && [ "$minor" -ge 10 ]; then
|
||||||
|
echo "$cmd"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
PY=$(find_python) || {
|
||||||
|
echo "[X] Python 3.10+ not found. Install it and re-run this script." >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
if [ ! -x "$VENV_DIR/bin/python" ]; then
|
||||||
|
echo "[*] Creating virtual environment in $VENV_DIR ..."
|
||||||
|
"$PY" -m venv "$VENV_DIR"
|
||||||
|
fi
|
||||||
|
|
||||||
|
VPY="$VENV_DIR/bin/python"
|
||||||
|
|
||||||
|
echo "[*] Installing dependencies ..."
|
||||||
|
"$VPY" -m pip install --disable-pip-version-check -q --upgrade pip >/dev/null
|
||||||
|
if ! "$VPY" -m pip install --disable-pip-version-check -q -r requirements.txt; then
|
||||||
|
echo "[!] PyPI install failed. Retrying via runflare mirror ..."
|
||||||
|
"$VPY" -m pip install --disable-pip-version-check -q -r requirements.txt \
|
||||||
|
-i https://mirror-pypi.runflare.com/simple/ \
|
||||||
|
--trusted-host mirror-pypi.runflare.com
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -f "config.json" ]; then
|
||||||
|
echo "[*] No config.json found — launching setup wizard ..."
|
||||||
|
"$VPY" setup.py
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "[*] Starting MasterHttpRelayVPN ..."
|
||||||
|
echo
|
||||||
|
exec "$VPY" main.py "$@"
|
||||||
Reference in New Issue
Block a user