using System.Net;
using System.Net.Http;
using System.Net.Quic;
using System.Net.Security;
using System.Net.Sockets;
using System.Security.Authentication;
using Fleet.Core;
using MoShiro.Protocol;
namespace MoShiro.Measurement;
/// http 请求的结果,编码与 Backend 的 HttpOutcome 一致。
internal static class HttpOutcomes
{
public const int Ok = 0;
public const int Timeout = 1;
public const int Reset = 2;
public const int Tls = 3;
public const int Other = 4;
public const int Dns = 5;
public const int BadStatus = 6;
public const int Unsupported = 7;
public const int NotAllowed = 8;
}
/// 解析出的地址都不在本地允许的网段内。
internal sealed class TargetNotAllowedException : Exception
{
public TargetNotAllowedException(string host)
: base("no address of " + host + " is allowed by allowed_target_cidrs")
{
}
}
///
/// 单个 Agent 去请求一个 URL(HTTP/1.1、HTTP/2 或 HTTP/3,不回退),记录 DNS、TCP 建连、首字节、总耗时和状态码。
/// 每次请求新建连接,所以每次都包含完整的建连过程;频率按失败率自适应:目标一直应答就降到证明得了成功率的最低频率,
/// 出现失败升到最高频率,全部失败时指数退避。
///
public sealed class HttpDialer : BlockBase
{
private readonly DriverLog _log;
private readonly int _attempt;
private readonly int _ts;
private readonly int _w;
private readonly int _dns;
private readonly int _tcp;
private readonly int _ttfb;
private readonly int _total;
private readonly int _status;
private readonly int _proto;
private readonly int _outcome;
private readonly HashSet _loggedOutcomes = [];
private long _next;
public HttpDialer(BlockEnv env, BlockSpec spec)
: base(env, spec)
{
_log = new DriverLog(spec.Kind.FindStream("req")!, Side.Dialer, 1024);
_attempt = _log.Store.ColumnIndex("attempt");
_ts = _log.Store.ColumnIndex("ts");
_w = _log.Store.ColumnIndex("w");
_dns = _log.Store.ColumnIndex("dns_us");
_tcp = _log.Store.ColumnIndex("tcp_us");
_ttfb = _log.Store.ColumnIndex("ttfb_us");
_total = _log.Store.ColumnIndex("total_us");
_status = _log.Store.ColumnIndex("status");
_proto = _log.Store.ColumnIndex("proto");
_outcome = _log.Store.ColumnIndex("outcome");
}
public override long Produced => _log.Count;
public override async Task RunAsync(CancellationToken ct)
{
if (!TargetHost.TryParse(Env.Target, out _, out _, out Uri? url) || url is null)
{
Env.Log.Warn("http target is not a URL", ("task", Spec.Scope.Task?.Value));
await WaitUntilCancelledAsync(ct);
return;
}
AdaptiveRate? rate = CreateRequestRate();
try
{
if (rate is not null)
{
await PaceAsync(rate, async () =>
{
int weight = rate.Weight;
long sent = Env.Clock.NowUs();
rate.OnSend(sent);
bool ok = await RequestAsync(url, weight, ct);
if (ok)
{
long now = Env.Clock.NowUs();
rate.OnReply(sent, now, (now - sent) / 1000.0);
}
}, ct);
}
else
{
TimeSpan interval = TimeSpan.FromSeconds(60.0 / Spec.Int("per_minute"));
while (!ct.IsCancellationRequested)
{
long start = Env.Clock.NowUs();
await RequestAsync(url, 1, ct);
TimeSpan wait = interval - TimeSpan.FromMicroseconds(Env.Clock.NowUs() - start);
if (wait > TimeSpan.Zero)
{
await Task.Delay(wait, Env.Time, ct);
}
}
}
}
catch (OperationCanceledException)
{
// 积木被停止
}
}
public override int Flush(FlushMode mode, long peerProduced = 0)
{
RawStream? stream = _log.Take(null, all: true);
if (stream is null)
{
return 0;
}
Ship(stream, "ts");
return 1;
}
///
/// 频率以「次/分钟」声明,换算成 Hz 交给 。请求间隔以秒到分钟计,所以窗口按分钟级放宽,
/// 并且不用 RTT 分位数的精度保护(样本太少,估计不出来):平稳时只由成功率的上界决定频率。
///
private AdaptiveRate? CreateRequestRate()
{
if (!Spec.Bool("adaptive"))
{
return null;
}
double max = Math.Max(Spec.Int("rate_max_per_min"), Spec.Int("rate_min_per_min")) / 60.0;
return new AdaptiveRate(new AdaptiveRateOptions
{
MinHz = Spec.Int("rate_min_per_min") / 60.0,
MaxHz = max,
TauPct = Spec.Number("tau_pct"),
RttGuard = false,
CertifySeconds = 900,
QuietCertifySeconds = 1200,
GraceSeconds = Math.Ceiling(Spec.Int("timeout_ms") / 1000.0) + 1,
DeadWindowSeconds = 120,
DeadMinSent = 3,
DeadMaxIntervalSeconds = 300,
ActiveLossCount = 2,
Prior = Env.LossPrior,
});
}
/// 发一次请求并记一行;返回这次算不算成功(有应答且状态码合格)。
private async Task RequestAsync(Uri url, int weight, CancellationToken ct)
{
string version = Spec.Text("version");
bool verify = Spec.Bool("tls_verify");
TimeSpan timeout = TimeSpan.FromMilliseconds(Spec.Int("timeout_ms"));
var phases = new Phases();
long start = Env.Clock.NowUs();
int outcome = HttpOutcomes.Other;
int status = 0;
int proto = 0;
long? ttfb = null;
long? total = null;
using var timeoutCts = new CancellationTokenSource(timeout, Env.Time);
using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(ct, timeoutCts.Token);
try
{
if (version == "h3" && !QuicConnection.IsSupported)
{
outcome = HttpOutcomes.Unsupported;
LogFirstFailure(outcome, url, version, null);
}
else
{
if (version == "h3")
{
await ResolveForQuicAsync(url.IdnHost, phases, linked.Token);
}
using HttpMessageInvoker invoker = CreateInvoker(version, verify, phases);
using var request = new HttpRequestMessage(new HttpMethod(Spec.Text("method")), url)
{
Version = version switch { "h1" => HttpVersion.Version11, "h2" => HttpVersion.Version20, _ => HttpVersion.Version30 },
VersionPolicy = HttpVersionPolicy.RequestVersionExact,
};
using HttpResponseMessage response = await invoker.SendAsync(request, linked.Token);
ttfb = Env.Clock.NowUs() - start;
status = (int)response.StatusCode;
proto = response.Version.Major switch { 3 => 3, 2 => 2, _ => 1 };
await ReadBodyAsync(response, (int)Spec.Int("read_kb") * 1024, linked.Token);
total = Env.Clock.NowUs() - start;
long expected = Spec.Int("ok_status");
outcome = (expected == 0 ? status < 400 : status == expected) ? HttpOutcomes.Ok : HttpOutcomes.BadStatus;
}
}
catch (OperationCanceledException) when (!ct.IsCancellationRequested)
{
outcome = HttpOutcomes.Timeout;
}
catch (OperationCanceledException)
{
return false;
}
catch (Exception ex) when (ex is HttpRequestException or IOException or SocketException or AuthenticationException or TargetNotAllowedException or QuicException)
{
outcome = Classify(ex);
LogFirstFailure(outcome, url, version, ex);
}
Record(start, weight, phases, ttfb, total, status, proto, outcome);
return outcome == HttpOutcomes.Ok;
}
private HttpMessageInvoker CreateInvoker(string version, bool verify, Phases phases)
{
var handler = new SocketsHttpHandler
{
UseProxy = false,
AllowAutoRedirect = false,
UseCookies = false,
AutomaticDecompression = DecompressionMethods.None,
};
// 设置了 ConnectCallback 的处理器不能用 HTTP/3(运行时用自己的 QUIC 连接,不经过回调):h3 只能靠请求前的解析和网段检查
if (version != "h3")
{
handler.ConnectCallback = (context, token) => ConnectAsync(context.DnsEndPoint, phases, token);
}
if (!verify)
{
#pragma warning disable CA5359 // 操作员明确关闭证书校验(tls_verify=false):只测可达性和耗时
handler.SslOptions = new SslClientAuthenticationOptions { RemoteCertificateValidationCallback = (_, _, _, _) => true };
#pragma warning restore CA5359
}
return new HttpMessageInvoker(handler, disposeHandler: true);
}
/// 自己解析和连接:每个地址都要在允许的网段里,DNS 和 TCP 建连的耗时分别记下来。
private async ValueTask ConnectAsync(DnsEndPoint endpoint, Phases phases, CancellationToken ct)
{
IPAddress[] addresses = await ResolveAsync(endpoint.Host, phases, ct);
long start = Env.Clock.NowUs();
Exception? last = null;
foreach (IPAddress address in addresses)
{
var socket = new Socket(address.AddressFamily, SocketType.Stream, ProtocolType.Tcp) { NoDelay = true };
try
{
await socket.ConnectAsync(new IPEndPoint(address, endpoint.Port), ct);
phases.TcpUs = Env.Clock.NowUs() - start;
return new NetworkStream(socket, ownsSocket: true);
}
catch (SocketException ex)
{
socket.Dispose();
last = ex;
}
}
throw last ?? new SocketException((int)SocketError.HostNotFound);
}
/// HTTP/3 由运行时自己建 UDP 连接,没法接管;先解析并检查一遍,再让它去连。
private async Task ResolveForQuicAsync(string host, Phases phases, CancellationToken ct) => _ = await ResolveAsync(host, phases, ct);
private async Task ResolveAsync(string host, Phases phases, CancellationToken ct)
{
long start = Env.Clock.NowUs();
IPAddress[] all = IPAddress.TryParse(host, out IPAddress? literal) ? [literal] : await Dns.GetHostAddressesAsync(host, ct);
phases.DnsUs = Env.Clock.NowUs() - start;
IPAddress[] allowed = [.. all.Where(a => Env.AllowAddress?.Invoke(a) ?? true)];
return allowed.Length > 0 ? allowed : throw new TargetNotAllowedException(host);
}
private static async Task ReadBodyAsync(HttpResponseMessage response, int limit, CancellationToken ct)
{
if (limit <= 0)
{
return;
}
await using Stream body = await response.Content.ReadAsStreamAsync(ct);
byte[] buffer = new byte[Math.Min(limit, 16 * 1024)];
int read = 0;
while (read < limit)
{
int n = await body.ReadAsync(buffer.AsMemory(0, Math.Min(buffer.Length, limit - read)), ct);
if (n == 0)
{
break;
}
read += n;
}
}
/// 每种失败结果只记第一次(带原因),保留给事后审计,不会随请求频率刷屏。
private void LogFirstFailure(int outcome, Uri url, string version, Exception? ex)
{
if (!_loggedOutcomes.Add(outcome))
{
return;
}
Env.Log.Warn("http request failed", ("task", Spec.Scope.Task?.Value), ("outcome", outcome), ("version", version), ("host", url.Host),
("quic_supported", version == "h3" ? QuicConnection.IsSupported : null), ("err", ex is null ? "System.Net.Quic is not available" : ex.GetType().Name + ": " + (ex.InnerException?.Message ?? ex.Message)));
}
private static int Classify(Exception ex)
{
for (Exception? e = ex; e is not null; e = e.InnerException)
{
switch (e)
{
case TargetNotAllowedException:
return HttpOutcomes.NotAllowed;
case AuthenticationException:
return HttpOutcomes.Tls;
case SocketException { SocketErrorCode: SocketError.HostNotFound or SocketError.TryAgain or SocketError.NoData }:
return HttpOutcomes.Dns;
case SocketException { SocketErrorCode: SocketError.ConnectionRefused or SocketError.ConnectionReset }:
return HttpOutcomes.Reset;
case HttpRequestException { HttpRequestError: HttpRequestError.NameResolutionError }:
return HttpOutcomes.Dns;
case HttpRequestException { HttpRequestError: HttpRequestError.SecureConnectionError }:
return HttpOutcomes.Tls;
case HttpRequestException { HttpRequestError: HttpRequestError.VersionNegotiationError }:
return HttpOutcomes.Unsupported;
}
}
return HttpOutcomes.Other;
}
private void Record(long startUs, int weight, Phases phases, long? ttfb, long? total, int status, int proto, int outcome)
{
long key = _log.Store.Append();
_log.Store.Set(key, _attempt, _next++);
_log.Store.Set(key, _ts, startUs);
_log.Store.Set(key, _w, Math.Clamp(weight, 1, 65535));
_log.Store.Set(key, _outcome, outcome);
SetIf(key, _dns, phases.DnsUs);
SetIf(key, _tcp, phases.TcpUs);
SetIf(key, _ttfb, ttfb);
SetIf(key, _total, total);
if (status != 0)
{
_log.Store.Set(key, _status, status);
_log.Store.Set(key, _proto, proto);
}
}
private void SetIf(long key, int column, long? value)
{
if (value is { } v)
{
_log.Store.Set(key, column, Math.Clamp(v, 0, uint.MaxValue));
}
}
/// 一次请求里 DNS 和 TCP 建连用了多久(连接回调里填)。
private sealed class Phases
{
public long? DnsUs { get; set; }
public long? TcpUs { get; set; }
}
}