Enhance MTProxy checker by introducing a new -probe mode deep-strict for stricter response validation. Update README and Docker documentation to clarify probing modes, exit codes, and the new MTPROXY_TDLIB_HELPER and MTPROXY_TDLIB_TIMEOUT environment variables for external helper integration. Refactor connection handling and error reporting to improve robustness and clarity in response verification.

cmd/mtproxy_checker/main.go

@@ -22,7 +22,7 @@ func main() {
 
 func run() int {
 	timeout := flag.Duration("timeout", 15*time.Second, "overall TCP/handshake timeout")
-	probe := flag.String("probe", "fast", "fast: handshake+init, Telethon-style post-init wait (no immediate close); deep: req_pq/resPQ via DC")
+	probe := flag.String("probe", "fast", "fast: handshake+init, Telethon-style post-init wait (no immediate close); deep: req_pq + any unencrypted DC reply; deep-strict: must be resPQ")
 	dcID := flag.Int("dc-id", 2, "Telegram DC id (signed int16) embedded in MTProxy header (ignored if -dc-ids is set)")
 	dcIDsFlag := flag.String("dc-ids", "", "comma-separated DC ids to probe in order (e.g. 1,2,3,4,5); OK if any succeeds; timeout is per-DC")
 	server := flag.String("server", "", "proxy hostname (if not using tg:// positional)")

cmd/mtproxy_checkerd/main.go

@@ -39,6 +39,8 @@ type config struct {
 	dcIDs           []int16
 	probe           checker.ProbeMode
 	allowedPrefixes []netip.Prefix
+	tdlibHelper     string
+	tdlibTimeout    time.Duration
 }
 
 func loadConfig() (*config, error) {
@@ -90,6 +92,15 @@ func loadConfig() (*config, error) {
 		return nil, err
 	}
 	probe := checker.ParseProbe(os.Getenv("MTPROXY_PROBE"))
+	tdlibHelper := strings.TrimSpace(os.Getenv("MTPROXY_TDLIB_HELPER"))
+	tdlibTimeoutStr := strings.TrimSpace(os.Getenv("MTPROXY_TDLIB_TIMEOUT"))
+	if tdlibTimeoutStr == "" {
+		tdlibTimeoutStr = "45s"
+	}
+	tdlibTimeout, err := time.ParseDuration(tdlibTimeoutStr)
+	if err != nil || tdlibTimeout <= 0 {
+		return nil, fmt.Errorf("MTPROXY_TDLIB_TIMEOUT: invalid duration %q", tdlibTimeoutStr)
+	}
 	return &config{
 		listFile:        listFile,
 		checkInterval:   interval,
@@ -98,6 +109,8 @@ func loadConfig() (*config, error) {
 		dcIDs:           dcIDs,
 		probe:           probe,
 		allowedPrefixes: prefixes,
+		tdlibHelper:     tdlibHelper,
+		tdlibTimeout:    tdlibTimeout,
 	}, nil
 }
 
@@ -108,14 +121,32 @@ type dcProbeResult struct {
 	Error    string `json:"error,omitempty"`
 }
 
+type standardProbe struct {
+	OK         bool   `json:"ok"`
+	ExitCode   int    `json:"exit_code"`
+	Error      string `json:"error,omitempty"`
+	ParseError string `json:"parse_error,omitempty"`
+}
+
+type tdlibProbe struct {
+	SkippedReason string `json:"skipped_reason,omitempty"`
+	Ran           bool   `json:"ran"`
+	OK            bool   `json:"ok"`
+	ExitCode      int    `json:"exit_code,omitempty"`
+	Error         string `json:"error,omitempty"`
+	DurationMs    int64  `json:"duration_ms,omitempty"`
+}
+
 type proxyEntry struct {
 	RawLine    string          `json:"raw_line"`
 	URL        string          `json:"url,omitempty"`
+	CheckedAt  string          `json:"checked_at,omitempty"`
+	Standard   standardProbe   `json:"standard"`
+	TDLib      tdlibProbe      `json:"tdlib"`
 	OK         bool            `json:"ok"`
 	ExitCode   int             `json:"exit_code"`
 	Error      string          `json:"error,omitempty"`
 	ParseError string          `json:"parse_error,omitempty"`
-	CheckedAt  string          `json:"checked_at,omitempty"`
 	DCs        []dcProbeResult `json:"dcs,omitempty"`
 }
 
@@ -185,9 +216,13 @@ func aggregateExitFromDCs(dcs []dcProbeResult) int {
 	return 1
 }
 
-func checkOneLine(ctx context.Context, line string, dcIDs []int16, perDCTimeout time.Duration, probe checker.ProbeMode) proxyEntry {
+func checkOneLine(ctx context.Context, line string, dcIDs []int16, perDCTimeout time.Duration, probe checker.ProbeMode, tdlibHelper string, tdlibTimeout time.Duration) (ent proxyEntry) {
 	now := time.Now().UTC().Format(time.RFC3339)
-	ent := proxyEntry{RawLine: line, CheckedAt: now}
+	ent = proxyEntry{RawLine: line, CheckedAt: now}
+	defer func() {
+		ent.Standard = standardProbe{OK: ent.OK, ExitCode: ent.ExitCode, Error: ent.Error, ParseError: ent.ParseError}
+		ent.TDLib = runTDLibBlock(line, tdlibHelper, tdlibTimeout, probe, ent)
+	}()
 	if len(dcIDs) == 0 {
 		ent.ExitCode = 2
 		ent.Error = "no DC ids configured"
@@ -210,8 +245,8 @@ func checkOneLine(ctx context.Context, line string, dcIDs []int16, perDCTimeout
 	}
 	if len(dcIDs) == 1 {
 		dcCtx, cancel := context.WithTimeout(ctx, perDCTimeout)
-		defer cancel()
 		err = checker.Check(dcCtx, t.Host, t.Port, parsed, dcIDs[0], &checker.Options{Probe: probe})
+		cancel()
 		code, msg := checkresult.Classify(err)
 		ent.ExitCode = code
 		ent.OK = err == nil
@@ -262,6 +297,8 @@ func runCycle(cfg *config, st *store) {
 			ParseError: err.Error(),
 			Error:      err.Error(),
 			CheckedAt:  finished.Format(time.RFC3339),
+			Standard:   standardProbe{OK: false, ExitCode: 2, Error: err.Error(), ParseError: err.Error()},
+			TDLib:      tdlibProbe{SkippedReason: "cycle aborted (list file error)", Ran: false},
 		}}, finished, time.Now().Add(cfg.checkInterval))
 		log.Printf("read list file: %v", err)
 		return
@@ -272,8 +309,11 @@ func runCycle(cfg *config, st *store) {
 		if len(cfg.dcIDs) > 1 {
 			total = cfg.checkTimeout * time.Duration(len(cfg.dcIDs))
 		}
+		if cfg.probe == checker.ProbeFast && strings.TrimSpace(cfg.tdlibHelper) != "" {
+			total += cfg.tdlibTimeout
+		}
 		ctx, cancel := context.WithTimeout(context.Background(), total)
-		ent := checkOneLine(ctx, line, cfg.dcIDs, cfg.checkTimeout, cfg.probe)
+		ent := checkOneLine(ctx, line, cfg.dcIDs, cfg.checkTimeout, cfg.probe, cfg.tdlibHelper, cfg.tdlibTimeout)
 		cancel()
 		entries = append(entries, ent)
 	}
@@ -353,7 +393,11 @@ func run() error {
 
 	errCh := make(chan error, 1)
 	go func() {
-		log.Printf("listening on %s, list=%s interval=%s dcs=%v", cfg.httpAddr, cfg.listFile, cfg.checkInterval, cfg.dcIDs)
+		if cfg.tdlibHelper != "" {
+			log.Printf("listening on %s, list=%s interval=%s dcs=%v tdlib_helper=%s tdlib_timeout=%s", cfg.httpAddr, cfg.listFile, cfg.checkInterval, cfg.dcIDs, cfg.tdlibHelper, cfg.tdlibTimeout)
+		} else {
+			log.Printf("listening on %s, list=%s interval=%s dcs=%v", cfg.httpAddr, cfg.listFile, cfg.checkInterval, cfg.dcIDs)
+		}
 		errCh <- srv.ListenAndServe()
 	}()
 

cmd/mtproxy_checkerd/tdlib_block.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"time"
+
+	"mtproxy_checker/internal/checker"
+	"mtproxy_checker/internal/tdlibexec"
+)
+
+func runTDLibBlock(proxyLine, helper string, timeout time.Duration, probe checker.ProbeMode, ent proxyEntry) tdlibProbe {
+	if probe != checker.ProbeFast {
+		return tdlibProbe{SkippedReason: "tdlib runs only with probe fast (or empty MTPROXY_PROBE)", Ran: false}
+	}
+	if ent.ParseError != "" {
+		return tdlibProbe{SkippedReason: "skipped: standard parse error", Ran: false}
+	}
+	if strings.TrimSpace(helper) == "" {
+		return tdlibProbe{SkippedReason: "MTPROXY_TDLIB_HELPER not set", Ran: false}
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	res, err := tdlibexec.Run(ctx, helper, proxyLine)
+	if err != nil {
+		if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
+			return tdlibProbe{Ran: true, OK: false, ExitCode: 4, Error: err.Error(), DurationMs: res.DurationMs}
+		}
+		return tdlibProbe{Ran: true, OK: false, ExitCode: 1, Error: err.Error(), DurationMs: res.DurationMs}
+	}
+	return tdlibProbe{Ran: true, OK: res.OK, ExitCode: res.ExitCode, Error: res.Error, DurationMs: res.DurationMs}
+}

cmd/tdlib_ping/README.md

@@ -0,0 +1,51 @@
+# tdlib_ping — официальный JSON-интерфейс TDLib
+
+Бинарник вызывает **`td_create_client_id` / `td_send` / `td_receive` / `td_execute`** из [`td/telegram/td_json_client.h`](https://github.com/tdlib/td/blob/master/td/telegram/td_json_client.h) (рекомендуемый multi-client API), без Node.js.
+
+Контракт stdout — одна строка JSON для `mtproxy_checkerd`:
+
+`{"ok":true,"error":"","exit_code":0}`
+
+## Сборка
+
+Linux (Debian/Ubuntu), установите заголовки и библиотеку, например:
+
+```bash
+sudo apt-get install -y libtdjson-dev   # имя пакета может отличаться по дистрибутиву
+export CGO_ENABLED=1
+go build -tags=tdlib -o tdlib_ping ./cmd/tdlib_ping
+```
+
+macOS: `brew install tdlib`, затем при необходимости `export PKG_CONFIG_PATH=...` если `pkg-config --libs tdlib` находит `-ltdjson`.
+
+Без `libtdjson` линковка завершится ошибкой — это ожидаемо.
+
+Сборка **без** тега (как часть `go build ./...`):
+
+```bash
+go build ./cmd/tdlib_ping
+```
+
+получится заглушка, которая печатает JSON с `exit_code: 2` и пояснением — чтобы репозиторий собирался без TDLib.
+
+## Переменные окружения
+
+| Переменная | По умолчанию | Назначение |
+|------------|--------------|------------|
+| `MTPROXY_TD_API_ID` | `12345` | Замените на значение с [my.telegram.org](https://my.telegram.org) |
+| `MTPROXY_TD_API_HASH` | демо-строка из примера TDLib | Замените на свой `api_hash` |
+| `MTPROXY_TDLIB_PING_MS` | `45000` | Общий дедлайн цикла (мс) |
+| `MTPROXY_TDLIB_DATABASE_DIR` | пусто | Если задан — постоянный каталог для `database_directory` / `files_directory` (подкаталоги `db/` и `files/` создаются автоматически). Иначе — временный каталог на каждый запуск |
+| `MTPROXY_TD_USE_TEST_DC` | пусто | Если `1` — `use_test_dc: true` в `setTdlibParameters` (иногда помогает обойти ожидание телефона на «чистой» БД) |
+
+## Запуск
+
+```bash
+./tdlib_ping 'tg://proxy?server=HOST&port=PORT&secret=HEX'
+```
+
+Если **`MTPROXY_TDLIB_DATABASE_DIR`** не задан, каталоги БД TDLib создаются под уникальным префиксом во временном каталоге на каждый запуск (и удаляются после выхода).
+
+## Ограничение
+
+При первом запуске с «чистой» БД TDLib может перейти в **`authorizationStateWaitPhoneNumber`**. Тогда helper завершится с ошибкой в JSON — нужен уже проинициализированный `database_directory` или рабочий сценарий входа. Для типичного мониторинга MTProxy на выделенной машине обычно достаточно повторных запусков с фиксированным каталогом (`MTPROXY_TDLIB_DATABASE_DIR`, см. таблицу выше).

cmd/tdlib_ping/stub.go

@@ -0,0 +1,19 @@
+//go:build !tdlib || !cgo
+
+package main
+
+import (
+	"encoding/json"
+	"os"
+)
+
+func main() {
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetEscapeHTML(false)
+	_ = enc.Encode(map[string]interface{}{
+		"ok":         false,
+		"error":      "tdlib_ping: build with CGO and -tags=tdlib; requires libtdjson (see cmd/tdlib_ping/README.md)",
+		"exit_code":  2,
+	})
+	os.Exit(2)
+}

cmd/tdlib_ping/tdlib.go

@@ -0,0 +1,292 @@
+//go:build tdlib && cgo
+
+package main
+
+/*
+#cgo LDFLAGS: -ltdjson
+#cgo CFLAGS: -I/usr/include -I/usr/local/include
+#include <stdlib.h>
+#include <td/telegram/td_json_client.h>
+*/
+import "C"
+
+import (
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+	"unsafe"
+
+	"mtproxy_checker/internal/parseurl"
+	"mtproxy_checker/internal/secret"
+)
+
+func main() {
+	if len(os.Args) < 2 {
+		outJSON(false, "usage: tdlib_ping <tg://proxy?...>", 2)
+	}
+	run(os.Args[1])
+}
+
+func outJSON(ok bool, msg string, code int) {
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetEscapeHTML(false)
+	_ = enc.Encode(map[string]interface{}{
+		"ok": ok, "error": msg, "exit_code": code,
+	})
+	if ok {
+		os.Exit(0)
+	}
+	switch code {
+	case 1:
+		os.Exit(1)
+	case 4:
+		os.Exit(4)
+	default:
+		os.Exit(2)
+	}
+}
+
+func run(proxyURL string) {
+	deadlineMs, _ := strconv.Atoi(os.Getenv("MTPROXY_TDLIB_PING_MS"))
+	if deadlineMs <= 0 {
+		deadlineMs = 45000
+	}
+	deadline := time.Now().Add(time.Duration(deadlineMs) * time.Millisecond)
+
+	t, err := parseurl.ParseTGProxy(proxyURL)
+	if err != nil {
+		outJSON(false, err.Error(), 1)
+	}
+	parsed, err := secret.Parse(t.Secret)
+	if err != nil {
+		outJSON(false, err.Error(), 1)
+	}
+	secHex := mtprotoSecretHex(parsed)
+
+	apiID := int64(12345)
+	if v := strings.TrimSpace(os.Getenv("MTPROXY_TD_API_ID")); v != "" {
+		if n, e := strconv.ParseInt(v, 10, 32); e == nil {
+			apiID = n
+		}
+	}
+	apiHash := strings.TrimSpace(os.Getenv("MTPROXY_TD_API_HASH"))
+	if apiHash == "" {
+		apiHash = "0123456789abcdef0123456789abcdef"
+	}
+	useTest := strings.TrimSpace(os.Getenv("MTPROXY_TD_USE_TEST_DC")) == "1"
+
+	dbBase := strings.TrimSpace(os.Getenv("MTPROXY_TDLIB_DATABASE_DIR"))
+	var cleanupTemp string
+	if dbBase == "" {
+		cleanupTemp, err = os.MkdirTemp("", "mtproxy_tdlib_")
+		if err != nil {
+			outJSON(false, err.Error(), 2)
+		}
+		dbBase = cleanupTemp
+		defer func() { _ = os.RemoveAll(cleanupTemp) }()
+	}
+
+	verb := `{"@type":"setLogVerbosityLevel","new_verbosity_level":1}`
+	if cs := C.CString(verb); cs != nil {
+		_ = C.td_execute(cs)
+		C.free(unsafe.Pointer(cs))
+	}
+
+	clientID := int(C.td_create_client_id())
+	defer closeClient(clientID)
+
+	if !waitAuthState(clientID, "authorizationStateWaitTdlibParameters", deadline) {
+		outJSON(false, "timeout waiting for authorizationStateWaitTdlibParameters", 4)
+	}
+
+	params := map[string]interface{}{
+		"@type":                    "setTdlibParameters",
+		"use_test_dc":              useTest,
+		"database_directory":       dbBase + "/db",
+		"files_directory":          dbBase + "/files",
+		"use_file_database":        true,
+		"use_chat_info_database":     true,
+		"use_message_database":       true,
+		"use_secret_chats":           true,
+		"api_id":                     apiID,
+		"api_hash":                   apiHash,
+		"system_language_code":       "en",
+		"device_model":               "mtproxy-tdlib-ping",
+		"application_version":        "1.0",
+	}
+	if err := os.MkdirAll(dbBase+"/db", 0o700); err != nil {
+		outJSON(false, err.Error(), 2)
+	}
+	if err := os.MkdirAll(dbBase+"/files", 0o700); err != nil {
+		outJSON(false, err.Error(), 2)
+	}
+	tdSend(clientID, params)
+
+	for time.Now().Before(deadline) {
+		ev := tdReceive(1.0)
+		if ev == nil {
+			continue
+		}
+		if typ, _ := ev["@type"].(string); typ == "error" {
+			outJSON(false, fmt.Sprintf("tdlib: %v", ev["message"]), 2)
+		}
+		switch authStateType(ev) {
+		case "authorizationStateWaitEncryptionKey":
+			tdSend(clientID, map[string]interface{}{
+				"@type": "checkDatabaseEncryptionKey", "encryption_key": "",
+			})
+		case "authorizationStateReady":
+			goto doProxy
+		case "authorizationStateWaitPhoneNumber":
+			outJSON(false, "TDLib authorizationStateWaitPhoneNumber: use a persistent MTPROXY_TDLIB_DATABASE_DIR after TDLib login, or set MTPROXY_TD_USE_TEST_DC=1", 2)
+		case "authorizationStateClosed":
+			outJSON(false, "unexpected authorizationStateClosed before ping", 2)
+		}
+	}
+	outJSON(false, "timeout in TDLib authorization", 4)
+
+doProxy:
+	addBody := map[string]interface{}{
+		"@type": "addProxy",
+		"server": t.Host, "port": t.Port, "enable": true,
+		"type": map[string]interface{}{
+			"@type": "proxyTypeMtproto", "secret": secHex,
+		},
+		"@extra": "addproxy",
+	}
+	tdSend(clientID, addBody)
+
+	var proxyID int64
+	for time.Now().Before(deadline) {
+		ev := tdReceive(1.0)
+		if ev == nil {
+			continue
+		}
+		if matchExtra(ev, "addproxy") {
+			if typ, _ := ev["@type"].(string); typ == "error" {
+				outJSON(false, fmt.Sprintf("addProxy: %v", ev["message"]), 2)
+			}
+			if typ, _ := ev["@type"].(string); typ == "proxy" {
+				if idf, ok := ev["id"].(float64); ok {
+					proxyID = int64(idf)
+					break
+				}
+			}
+			outJSON(false, fmt.Sprintf("addProxy unexpected: %#v", ev), 2)
+		}
+	}
+	if proxyID == 0 {
+		outJSON(false, "timeout waiting for addProxy response", 4)
+	}
+
+	tdSend(clientID, map[string]interface{}{
+		"@type": "pingProxy", "proxy_id": proxyID, "@extra": "pingpx",
+	})
+
+	for time.Now().Before(deadline) {
+		ev := tdReceive(1.0)
+		if ev == nil {
+			continue
+		}
+		if matchExtra(ev, "pingpx") {
+			if typ, _ := ev["@type"].(string); typ == "ok" {
+				outJSON(true, "", 0)
+			}
+			if typ, _ := ev["@type"].(string); typ == "error" {
+				outJSON(false, fmt.Sprintf("pingProxy: %v", ev["message"]), 2)
+			}
+			outJSON(false, fmt.Sprintf("pingProxy unexpected: %#v", ev), 2)
+		}
+	}
+	outJSON(false, "timeout waiting for pingProxy", 4)
+}
+
+func mtprotoSecretHex(p *secret.Parsed) string {
+	var full []byte
+	switch p.Kind {
+	case secret.KindDD:
+		full = append([]byte{0xdd}, p.Key...)
+	case secret.KindEE:
+		full = append(append([]byte{0xee}, p.Key...), p.Domain...)
+	default:
+		return ""
+	}
+	return hex.EncodeToString(full)
+}
+
+func authStateType(m map[string]interface{}) string {
+	if m == nil {
+		return ""
+	}
+	if m["@type"] != "updateAuthorizationState" {
+		return ""
+	}
+	v, _ := m["authorization_state"].(map[string]interface{})
+	if v == nil {
+		return ""
+	}
+	t, _ := v["@type"].(string)
+	return t
+}
+
+func matchExtra(m map[string]interface{}, want string) bool {
+	if m == nil {
+		return false
+	}
+	ext, ok := m["@extra"].(string)
+	return ok && ext == want
+}
+
+func waitAuthState(clientID int, want string, deadline time.Time) bool {
+	for time.Now().Before(deadline) {
+		ev := tdReceive(1.0)
+		if ev == nil {
+			continue
+		}
+		if authStateType(ev) == want {
+			return true
+		}
+	}
+	return false
+}
+
+func tdSend(clientID int, v interface{}) {
+	b, err := json.Marshal(v)
+	if err != nil {
+		outJSON(false, err.Error(), 2)
+	}
+	cs := C.CString(string(b))
+	defer C.free(unsafe.Pointer(cs))
+	C.td_send(C.int(clientID), cs)
+}
+
+func tdReceive(maxSec float64) map[string]interface{} {
+	cs := C.td_receive(C.double(maxSec))
+	if cs == nil {
+		return nil
+	}
+	s := C.GoString(cs)
+	var m map[string]interface{}
+	if err := json.Unmarshal([]byte(s), &m); err != nil {
+		return nil
+	}
+	return m
+}
+
+func closeClient(clientID int) {
+	tdSend(clientID, map[string]interface{}{"@type": "close"})
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		ev := tdReceive(0.5)
+		if ev == nil {
+			continue
+		}
+		if authStateType(ev) == "authorizationStateClosed" {
+			break
+		}
+	}
+}