diff --git a/.gitignore b/.gitignore index 222cdabf2..2cc93a052 100644 --- a/.gitignore +++ b/.gitignore @@ -9,12 +9,18 @@ /Distribution/bsdtar /Distribution/Configuration/antimacro.json /Distribution/Configuration/assistants.json +/Distribution/Configuration/auto-denylist.json /Distribution/Configuration/bans.json /Distribution/Configuration/blocklist.json /Distribution/Configuration/crowdsec.json /Distribution/Configuration/expansion.json +/Distribution/Configuration/ip-allowlist*.txt +/Distribution/Configuration/ip-allowlist*.txt.tmp /Distribution/Configuration/ip-blocklist.txt /Distribution/Configuration/ip-blocklist.txt.tmp +/Distribution/Configuration/login-allowlist.json +/Distribution/Configuration/login-allowlist.txt +/Distribution/Configuration/login-allowlist.txt.tmp /Distribution/Configuration/modernuo.json /Distribution/Configuration/email-settings.json /Distribution/Configuration/throttles.json diff --git a/CLAUDE.md b/CLAUDE.md index 0c41d2087..7f24661a4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -48,6 +48,7 @@ Apply these when writing or reviewing `.cs` files under `Projects/`. | Server lifecycle & bootstrap phases (Configure/ConfigurePrompts/Initialize) | `dev-docs/server-lifecycle.md` | | Configuration system | `dev-docs/configuration.md` | | Networking & packets | `dev-docs/networking-packets.md` | +| IP bans, blocklists & allowlists (incl. unblocking a player) | `dev-docs/ip-bans-and-allowlists.md` | | Region system | `dev-docs/regions.md` | | String handling & ValueStringBuilder | `dev-docs/string-handling.md` | | RunUO migration (overview) | `dev-docs/runuo-migration-docs/00-overview.md` | diff --git a/Projects/Server.Tests/Tests/Network/Bans/BanChannelTests.cs b/Projects/Server.Tests/Tests/Network/Bans/BanChannelTests.cs index a48647152..5eb1895d8 100644 --- a/Projects/Server.Tests/Tests/Network/Bans/BanChannelTests.cs +++ b/Projects/Server.Tests/Tests/Network/Bans/BanChannelTests.cs @@ -60,6 +60,29 @@ public class BanChannelTests Assert.Single(good.Reports); // the throwing reporter does not block the others } + [Fact] + public void Report_SuppressedForExemptAddress() + { + var reporter = new FakeReporter(); + BanChannel.ConfigureForTesting([reporter]); + + var exempt = IPAddress.Parse("203.0.113.7"); + BanChannel.IsExempt = (ip, _) => ip.Equals(exempt); + + try + { + BanChannel.Report(exempt, TimeSpan.FromHours(1), "rate-limit"); + Assert.Empty(reporter.Reports); // escalation suppressed; the local gate already acted + + BanChannel.Report(IPAddress.Parse("203.0.113.8"), TimeSpan.FromHours(1), "rate-limit"); + Assert.Single(reporter.Reports); // everyone else is still contributed + } + finally + { + BanChannel.IsExempt = null; + } + } + [Fact] public void Retract_ReachesRetractCapableReporters() { diff --git a/Projects/Server.Tests/Tests/Network/ForeignProtocolTests.cs b/Projects/Server.Tests/Tests/Network/ForeignProtocolTests.cs new file mode 100644 index 000000000..95a3c4114 --- /dev/null +++ b/Projects/Server.Tests/Tests/Network/ForeignProtocolTests.cs @@ -0,0 +1,116 @@ +/************************************************************************* + * ModernUO * + * Copyright 2019-2026 - ModernUO Development Team * + * Email: hi@modernuo.com * + * File: ForeignProtocolTests.cs * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +using System.Text; +using Server.Network; +using Xunit; + +namespace Server.Tests.Network; + +public class ForeignProtocolTests +{ + private static ForeignProtocolMatch Identify(byte[] bytes, out ForeignProtocolKind kind) => + ForeignProtocol.Identify(bytes, out kind); + + private static byte[] Ascii(string s) => Encoding.ASCII.GetBytes(s); + + [Theory] + [InlineData("GET / HTTP/1.1\r\n")] + [InlineData("POST /a HTTP/1.1\r\n")] + [InlineData("HEAD / HTTP/1.0\r\n")] + [InlineData("OPTIONS * HTTP/1.1\r\n")] + [InlineData("CONNECT host:443 HTTP/1.1\r\n")] + [InlineData("DELETE /x HTTP/1.1\r\n")] + public void Http_requests_are_identified(string request) + { + Assert.Equal(ForeignProtocolMatch.Confirmed, Identify(Ascii(request), out var kind)); + Assert.Equal(ForeignProtocolKind.Http, kind); + } + + [Fact] + public void Ssh_banner_is_identified() + { + Assert.Equal(ForeignProtocolMatch.Confirmed, Identify(Ascii("SSH-2.0-OpenSSH_9.6"), out var kind)); + Assert.Equal(ForeignProtocolKind.Ssh, kind); + } + + [Fact] + public void Tls_client_hello_is_identified() + { + // handshake, TLS 1.2 record, length 0x0100, ClientHello + byte[] hello = [0x16, 0x03, 0x03, 0x01, 0x00, 0x01, 0x00, 0x00, 0xFC]; + + Assert.Equal(ForeignProtocolMatch.Confirmed, Identify(hello, out var kind)); + Assert.Equal(ForeignProtocolKind.Tls, kind); + } + + [Fact] + public void Tls_prefix_without_a_client_hello_is_not_foreign() + { + // A seed can spell 0x16 0x03 0x0? -- the address 22.3.x.x. Byte five is a packet id, not a + // handshake type, so it must fall through to normal parsing. + byte[] seedThenLogin = [0x16, 0x03, 0x03, 0x04, 0x00, 0x80, 0x00, 0x00]; + + Assert.Equal(ForeignProtocolMatch.None, Identify(seedThenLogin, out var kind)); + Assert.Equal(ForeignProtocolKind.None, kind); + } + + [Fact] + public void Seed_that_spells_an_http_method_falls_through() + { + // Seed 0x47455420 spells "GET ". The next byte is the 0x80 login packet id, not printable, so this + // is a real client and must not be flagged. + byte[] seedThenLogin = [(byte)'G', (byte)'E', (byte)'T', (byte)' ', 0x80, 0x00, 0x3A, 0x00]; + + Assert.Equal(ForeignProtocolMatch.None, Identify(seedThenLogin, out _)); + } + + [Theory] + [InlineData(0x80)] // login request + [InlineData(0x91)] // game server login + [InlineData(0xEF)] // new-style seed packet + public void Ordinary_uo_openings_are_not_foreign(byte secondPacketId) + { + byte[] buffer = [0x7F, 0x00, 0x00, 0x01, secondPacketId, 0x00, 0x00, 0x00]; + + Assert.Equal(ForeignProtocolMatch.None, Identify(buffer, out _)); + } + + [Fact] + public void Encrypted_login_is_not_mistaken_for_a_foreign_protocol() + { + // A legitimate client with encryption on when the shard expects none. The login cipher is a + // byte-for-byte XOR, so this is noise of exactly the right length. + byte[] buffer = [0x7F, 0x00, 0x00, 0x01, 0xC3, 0x9A, 0x04, 0xE1, 0x55, 0xB2]; + + Assert.Equal(ForeignProtocolMatch.None, Identify(buffer, out _)); + } + + [Fact] + public void Prefix_match_without_enough_bytes_waits() + { + // Framing is never assumed: "GET " split from its request line must wait. + Assert.Equal(ForeignProtocolMatch.Incomplete, Identify(Ascii("GET "), out _)); + Assert.Equal(ForeignProtocolMatch.Incomplete, Identify(Ascii("GET /"), out _)); + } + + [Fact] + public void Too_few_bytes_to_match_a_prefix_is_not_foreign() + { + // Under four bytes the caller's own short-read handling applies. + Assert.Equal(ForeignProtocolMatch.None, Identify(Ascii("GE"), out _)); + Assert.Equal(ForeignProtocolMatch.None, Identify([], out _)); + } +} diff --git a/Projects/Server.Tests/Tests/PropertyList/ObjectPropertyListReentrancyTests.cs b/Projects/Server.Tests/Tests/PropertyList/ObjectPropertyListReentrancyTests.cs index 8da5f748a..a30e9c9e3 100644 --- a/Projects/Server.Tests/Tests/PropertyList/ObjectPropertyListReentrancyTests.cs +++ b/Projects/Server.Tests/Tests/PropertyList/ObjectPropertyListReentrancyTests.cs @@ -8,6 +8,8 @@ namespace Server.Tests; /// hole is evaluated while it is live. A Reset()/Dispose() landing in that window used to leave the /// next Append* spanning a null array: ArgumentNullException, parameter "array". /// +// Sequential: building a list rents from STArrayPool, which is not thread-safe. +[Collection("Sequential Server Tests")] public class ObjectPropertyListReentrancyTests { // Stands in for a property getter that invalidates while its own tooltip is being built. @@ -52,6 +54,7 @@ public class ObjectPropertyListReentrancyTests /// The guard is per-list, so nested builds (a GetProperties override that reads another entity's /// PropertyList) cannot unguard the outer one the way a single shared slot would. /// +[Collection("Sequential Server Tests")] public class ObjectPropertyListNestedBuildTests { [Fact] diff --git a/Projects/Server/Network/Bans/BanChannel.cs b/Projects/Server/Network/Bans/BanChannel.cs index 6d4ead9b1..b682158d0 100644 --- a/Projects/Server/Network/Bans/BanChannel.cs +++ b/Projects/Server/Network/Bans/BanChannel.cs @@ -103,8 +103,25 @@ public static class BanChannel } /// Fans a locally-decided ban out to every reporter. Non-blocking; never throws. + /// + /// Optional content-supplied exemption; true drops the contribution before any reporter sees it. This + /// withholds escalation only — the gate that reached the verdict has already acted. + /// + /// + /// An implementation that ignores reason would silently swallow manual bans. See + /// . + /// + public static Func IsExempt { get; set; } + public static void Report(IPAddress ip, TimeSpan ttl, string reason) { + var exempt = IsExempt; + if (exempt != null && exempt(ip, reason)) + { + logger.Debug("{Address} not contributed ('{Reason}'): exempt", ip, reason); + return; + } + var reporters = _reporters; for (var i = 0; i < reporters.Length; i++) { diff --git a/Projects/Server/Network/Bans/BanConfiguration.cs b/Projects/Server/Network/Bans/BanConfiguration.cs index 2a2af40cf..d678a6470 100644 --- a/Projects/Server/Network/Bans/BanConfiguration.cs +++ b/Projects/Server/Network/Bans/BanConfiguration.cs @@ -29,16 +29,24 @@ public static class BanConfiguration { private const string _path = "Configuration/bans.json"; - public static BanSettings Settings { get; private set; } + private static bool _loaded; + + /// + /// Never null: the accept and reap paths read this per connection, including before + /// has run (a harness driving NetState.Slice directly), so it starts at the record's defaults. + /// + public static BanSettings Settings { get; private set; } = new(); public static void Configure() { - // Idempotent: a second call must not re-deserialize or overwrite an operator's edits. - if (Settings != null) + // Idempotent; flagged rather than null-checked because Settings is non-null from the start. + if (_loaded) { return; } + _loaded = true; + var path = Path.Join(Core.BaseDirectory, _path); if (File.Exists(path)) @@ -73,4 +81,18 @@ public record BanSettings /// Duration reported for an auto-detected (rate-limit) ban. [JsonPropertyName("autoBanDuration")] public TimeSpan AutoBanDuration { get; set; } = TimeSpan.FromHours(4); + + /// + /// Whether behavioural detections are contributed to reporters. Those connections are disconnected either + /// way; this only controls escalation. + /// + /// + /// Keyed on bytes-received, never elapsed time: a connection that sent something and ran out of time is + /// far more likely a slow link than an attack. See dev-docs/ip-bans-and-allowlists.md. + /// + [JsonPropertyName("reportBadConnects")] + public bool ReportBadConnects { get; set; } = true; + + [JsonPropertyName("badConnectDuration")] + public TimeSpan BadConnectDuration { get; set; } = TimeSpan.FromHours(4); } diff --git a/Projects/Server/Network/Bans/BanReasons.cs b/Projects/Server/Network/Bans/BanReasons.cs new file mode 100644 index 000000000..16708f716 --- /dev/null +++ b/Projects/Server/Network/Bans/BanReasons.cs @@ -0,0 +1,48 @@ +/************************************************************************* + * ModernUO * + * Copyright 2019-2026 - ModernUO Development Team * + * Email: hi@modernuo.com * + * File: BanReasons.cs * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +namespace Server.Network.Bans; + +/// The reason slugs contributed through . Policy keys off these. +public static class BanReasons +{ + /// An operator banned this address explicitly. Never exempt, never auto-denied. + public const string Manual = "manual"; + + public const string RateLimit = "rate-limit"; + + /// Matched the reputation blocklist. Enforced by its own filter, not by behaviour. + public const string Blocklist = "blocklist"; + + /// Reaped without ever sending a byte. + public const string SilentConnect = "silent-connect"; + + /// Opened with a zero seed, which no real client sends. + public const string InvalidSeed = "invalid-seed"; + + /// Positively identified as another protocol entirely. See . + public const string ForeignProtocol = "foreign-protocol"; + + /// + /// Verdicts the shard reached by watching the connection. Only these may be exempted, and only these feed + /// the local denylist. + /// + /// + /// Opt-in rather than "everything except ", so a reason added later escalates normally + /// instead of silently inheriting an exemption. + /// + public static bool IsBehavioral(string reason) => + reason is RateLimit or SilentConnect or InvalidSeed or ForeignProtocol; +} diff --git a/Projects/Server/Network/ForeignProtocol.cs b/Projects/Server/Network/ForeignProtocol.cs new file mode 100644 index 000000000..059a69d3f --- /dev/null +++ b/Projects/Server/Network/ForeignProtocol.cs @@ -0,0 +1,146 @@ +/************************************************************************* + * ModernUO * + * Copyright 2019-2026 - ModernUO Development Team * + * Email: hi@modernuo.com * + * File: ForeignProtocol.cs * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +using System; + +namespace Server.Network; + +public enum ForeignProtocolKind +{ + None, + Http, + Tls, + Ssh +} + +public enum ForeignProtocolMatch +{ + None, + + /// A prefix matched, but more bytes are needed to be sure. + Incomplete, + + Confirmed +} + +/// +/// Identifies traffic that is positively some OTHER protocol (HTTP, TLS, SSH), rather than deciding whether +/// a connection is a good Ultima Online client. +/// +/// +/// The direction matters. A legitimate client with encryption enabled when the shard expects none sends a +/// structurally correct connection whose payload is noise, because LoginEncryption.ClientDecrypt is a +/// byte-for-byte stream XOR: length survives, content does not. So "unreadable" cannot mean "hostile", while +/// "speaks HTTP" safely can. Nothing here assumes arrival framing; see +/// dev-docs/ip-bans-and-allowlists.md. +/// +public static class ForeignProtocol +{ + private const int RequiredBytes = 8; + private const int MaxTlsRecordLength = 16384; + + public static ForeignProtocolMatch Identify(ReadOnlySpan buffer, out ForeignProtocolKind kind) + { + kind = ForeignProtocolKind.None; + + // Too little to match a prefix; the caller's own short-read handling covers it. + if (buffer.Length < 4) + { + return ForeignProtocolMatch.None; + } + + var candidate = MatchPrefix(buffer); + if (candidate == ForeignProtocolKind.None) + { + return ForeignProtocolMatch.None; + } + + if (buffer.Length < RequiredBytes) + { + return ForeignProtocolMatch.Incomplete; + } + + if (!Confirm(buffer, candidate)) + { + return ForeignProtocolMatch.None; + } + + kind = candidate; + return ForeignProtocolMatch.Confirmed; + } + + private static ForeignProtocolKind MatchPrefix(ReadOnlySpan buffer) + { + // TLS handshake record: content type 0x16, major version 3, minor version 0..4 (SSL 3.0 - TLS 1.3). + if (buffer[0] == 0x16 && buffer[1] == 0x03 && buffer[2] <= 0x04) + { + return ForeignProtocolKind.Tls; + } + + if (StartsWith(buffer, "SSH-")) + { + return ForeignProtocolKind.Ssh; + } + + // HTTP request methods. Four bytes only selects a candidate; Confirm checks the request line. + if (StartsWith(buffer, "GET ") || StartsWith(buffer, "POST") || StartsWith(buffer, "HEAD") || + StartsWith(buffer, "PUT ") || StartsWith(buffer, "OPTI") || StartsWith(buffer, "DELE") || + StartsWith(buffer, "CONN") || StartsWith(buffer, "TRAC") || StartsWith(buffer, "PATC")) + { + return ForeignProtocolKind.Http; + } + + return ForeignProtocolKind.None; + } + + private static bool Confirm(ReadOnlySpan buffer, ForeignProtocolKind candidate) + { + if (candidate == ForeignProtocolKind.Tls) + { + // A seed can collide with the 0x16 0x03 0x0? prefix (that is just the address 22.3.x.x), so + // require a ClientHello inside a plausible record. + var recordLength = (buffer[3] << 8) | buffer[4]; + return buffer[5] == 0x01 && recordLength is >= 4 and <= MaxTlsRecordLength; + } + + // A UO client's fifth byte is a packet id (0x80, 0x91, 0xEF), none of them printable, so requiring + // the request line to continue in ASCII lets a seed that spells "GET " fall through. + for (var i = 4; i < buffer.Length && i < 16; i++) + { + if (!IsPrintableAscii(buffer[i])) + { + return false; + } + } + + return true; + } + + private static bool IsPrintableAscii(byte value) => + value is >= 0x20 and <= 0x7E or (byte)'\r' or (byte)'\n' or (byte)'\t'; + + private static bool StartsWith(ReadOnlySpan buffer, string ascii) + { + for (var i = 0; i < ascii.Length; i++) + { + if (buffer[i] != (byte)ascii[i]) + { + return false; + } + } + + return true; + } +} diff --git a/Projects/Server/Network/NetState/NetState.Network.cs b/Projects/Server/Network/NetState/NetState.Network.cs index 5262c3c2e..7a17d8cd0 100644 --- a/Projects/Server/Network/NetState/NetState.Network.cs +++ b/Projects/Server/Network/NetState/NetState.Network.cs @@ -274,7 +274,7 @@ public partial class NetState { // Enqueue-only contribution; NOT added to the local firewall set (the limiter already // gates it here and the OS bouncer drops it at the kernel). - Bans.BanChannel.Report(remoteIP, Bans.BanConfiguration.Settings.AutoBanDuration, "rate-limit"); + Bans.BanChannel.Report(remoteIP, Bans.BanConfiguration.Settings.AutoBanDuration, Bans.BanReasons.RateLimit); } } else if (ConnectionFilters.ShouldDeny(remoteIP, out var deniedBy)) @@ -362,6 +362,18 @@ public partial class NetState // Socket must have finished the entire authentication process or be forcibly disconnected if (!ns.SentFirstPacket || !ns.Seeded) { + // Only the totally silent ones are evidence. A connection that sent SOME data and ran out of + // time is far more likely a slow link, and banning those makes the player retry, trip the + // rate limiter, and compound it into an hours-long ban. + if (!ns._receivedData && Bans.BanConfiguration.Settings.ReportBadConnects) + { + Bans.BanChannel.Report( + ns.Address, + Bans.BanConfiguration.Settings.BadConnectDuration, + Bans.BanReasons.SilentConnect + ); + } + ns.Disconnect(null); // Force immediate cleanup - these are unauthenticated connections @@ -538,6 +550,11 @@ public partial class NetState return; } + if (bytesReceived > 0) + { + ns._receivedData = true; + } + // Data is already committed to buffer by RingSocketManager // Decode if encryption is enabled ns.DecryptRecvBuffer(bytesReceived); diff --git a/Projects/Server/Network/NetState/NetState.cs b/Projects/Server/Network/NetState/NetState.cs index 05d8a8f16..7312603f7 100755 --- a/Projects/Server/Network/NetState/NetState.cs +++ b/Projects/Server/Network/NetState/NetState.cs @@ -60,6 +60,10 @@ public partial class NetState : IComparable, IValueLinkListNode, IValueLinkListNode, IValueLinkListNode, IValueLinkListNode. * + *************************************************************************/ + +using System.Net; +using Server.Network; +using Server.Network.Bans; +using Xunit; + +namespace Server.Tests.Network.AutoDenylists; + +// Static store, so every test resets it first. Addresses come from TEST-NET-2 (198.51.100.0/24). +// Sequential: the cap tests reach Sweep, which rents from STArrayPool, which is not thread-safe. +[Collection("Sequential UOContent Tests")] +public class AutoDenylistTests +{ + private const long DurationMs = 900_000; // 15 minutes, the shipped default + private const long Now = 1_000_000; + + private static void Reset(bool enabled = true, int maxEntries = 1024) => + AutoDenylist.LoadForTesting(enabled, DurationMs, maxEntries); + + [Fact] + public void Behavioral_detection_is_held_then_lapses() + { + Reset(); + var ip = IPAddress.Parse("198.51.100.10"); + + Assert.True(AutoDenylist.Hold(ip, BanReasons.InvalidSeed, Now)); + + Assert.True(AutoDenylist.IsDenied(ip, Now)); + Assert.True(AutoDenylist.IsDenied(ip, Now + DurationMs - 1)); + + // Expiry is decided on read, so the hold lapses without waiting for a sweep. + Assert.False(AutoDenylist.IsDenied(ip, Now + DurationMs)); + } + + [Fact] + public void Manual_and_list_verdicts_are_not_held() + { + Reset(); + var manual = IPAddress.Parse("198.51.100.11"); + var listed = IPAddress.Parse("198.51.100.12"); + + // A manual ban belongs in the firewall; a blocklist match is enforced by the blocklist's own filter. + Assert.False(AutoDenylist.Hold(manual, BanReasons.Manual, Now)); + Assert.False(AutoDenylist.Hold(listed, BanReasons.Blocklist, Now)); + + Assert.False(AutoDenylist.IsDenied(manual, Now)); + Assert.False(AutoDenylist.IsDenied(listed, Now)); + Assert.Equal(0, AutoDenylist.Count); + } + + [Fact] + public void Repeat_detection_extends_the_hold() + { + Reset(); + var ip = IPAddress.Parse("198.51.100.13"); + + AutoDenylist.Hold(ip, BanReasons.SilentConnect, Now); + AutoDenylist.Hold(ip, BanReasons.SilentConnect, Now + DurationMs - 1); + + Assert.True(AutoDenylist.IsDenied(ip, Now + DurationMs + 1)); // would have lapsed without the second + Assert.Equal(1, AutoDenylist.Count); // and did not add a duplicate + } + + [Fact] + public void Release_drops_the_hold_immediately() + { + Reset(); + var ip = IPAddress.Parse("198.51.100.14"); + + AutoDenylist.Hold(ip, BanReasons.RateLimit, Now); + AutoDenylist.Release(ip); + + Assert.False(AutoDenylist.IsDenied(ip, Now)); + } + + [Fact] + public void Cap_is_enforced_so_a_distinct_source_flood_cannot_grow_it() + { + Reset(maxEntries: 4); + + for (var i = 0; i < 10; i++) + { + AutoDenylist.Hold(IPAddress.Parse($"198.51.100.{100 + i}"), BanReasons.InvalidSeed, Now); + } + + Assert.Equal(4, AutoDenylist.Count); + + // The first four are still held; the rest were disconnected by their gate but not tracked. + Assert.True(AutoDenylist.IsDenied(IPAddress.Parse("198.51.100.100"), Now)); + Assert.False(AutoDenylist.IsDenied(IPAddress.Parse("198.51.100.109"), Now)); + } + + [Fact] + public void Reaching_the_cap_reclaims_lapsed_entries_first() + { + Reset(maxEntries: 2); + + AutoDenylist.Hold(IPAddress.Parse("198.51.100.20"), BanReasons.InvalidSeed, Now); + AutoDenylist.Hold(IPAddress.Parse("198.51.100.21"), BanReasons.InvalidSeed, Now); + + // Once those two have lapsed, a new detection sweeps them out rather than being refused. + var later = Now + DurationMs + 1; + Assert.True(AutoDenylist.Hold(IPAddress.Parse("198.51.100.22"), BanReasons.InvalidSeed, later)); + Assert.True(AutoDenylist.IsDenied(IPAddress.Parse("198.51.100.22"), later)); + } + + [Fact] + public void Disabled_store_denies_nobody() + { + Reset(enabled: false); + var ip = IPAddress.Parse("198.51.100.30"); + + Assert.False(AutoDenylist.Hold(ip, BanReasons.InvalidSeed, Now)); + Assert.False(AutoDenylist.IsDenied(ip, Now)); + } + + [Fact] + public void Null_address_is_handled() + { + Reset(); + + Assert.False(AutoDenylist.Hold(null, BanReasons.InvalidSeed, Now)); + Assert.False(AutoDenylist.IsDenied(null, Now)); + } +} diff --git a/Projects/UOContent.Tests/Tests/Network/BanExemptionsTests.cs b/Projects/UOContent.Tests/Tests/Network/BanExemptionsTests.cs new file mode 100644 index 000000000..7e930fbdf --- /dev/null +++ b/Projects/UOContent.Tests/Tests/Network/BanExemptionsTests.cs @@ -0,0 +1,122 @@ +/************************************************************************* + * ModernUO * + * Copyright 2019-2026 - ModernUO Development Team * + * Email: hi@modernuo.com * + * File: BanExemptionsTests.cs * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +using System.Net; +using System.Text; +using Server.Network; +using Server.Network.Bans; +using Xunit; + +namespace Server.Tests.Network.Exemptions; + +// Addresses come from TEST-NET-1 (192.0.2.0/24) to stay clear of the other ban test classes if xUnit runs +// them concurrently. +public class BanExemptionsTests +{ + private static readonly IPAddress _listed = IPAddress.Parse("192.0.2.10"); + private static readonly IPAddress _unlisted = IPAddress.Parse("192.0.2.11"); + + private static void WithFileAllowlist(string contents) => + FileAllowlist.LoadForTesting(BlocklistSnapshot.Build(Encoding.ASCII.GetBytes(contents), out _, out _)); + + private static void WithEmptyFileAllowlist() => FileAllowlist.LoadForTesting(BlocklistSnapshot.Empty); + + [Fact] + public void File_allowlist_exempts_behavioral_contributions() + { + WithFileAllowlist("192.0.2.10"); + + // Subtracting from the blocklist does nothing for behavioural detections, which never consult it. + Assert.True(BanExemptions.IsExempt(_listed, BanReasons.ForeignProtocol, NeverCalled)); + Assert.True(BanExemptions.IsExempt(_listed, BanReasons.RateLimit, NeverCalled)); + Assert.True(BanExemptions.IsExempt(_listed, BanReasons.SilentConnect, NeverCalled)); + Assert.True(BanExemptions.IsExempt(_listed, BanReasons.InvalidSeed, NeverCalled)); + } + + [Fact] + public void File_allowlist_covers_cidr_entries() + { + // Carve-outs are CIDRs, so a shared-CGNAT player is only covered if ranges work here. + WithFileAllowlist("192.0.2.0/24"); + + Assert.True(BanExemptions.IsExempt(_listed, BanReasons.RateLimit, NeverCalled)); + Assert.True(BanExemptions.IsExempt(IPAddress.Parse("192.0.2.254"), BanReasons.RateLimit, NeverCalled)); + Assert.False(BanExemptions.IsExempt(IPAddress.Parse("192.0.3.1"), BanReasons.RateLimit, AlwaysFalse)); + } + + [Fact] + public void Manual_bans_are_never_exempt_even_when_file_allowlisted() + { + WithFileAllowlist("192.0.2.10"); + + // An explicit decision outranks the operator's own carve-out, and must not cost a strike. + Assert.False(BanExemptions.IsExempt(_listed, BanReasons.Manual, NeverCalled)); + } + + [Fact] + public void Unopted_reasons_are_never_exempt() + { + WithFileAllowlist("192.0.2.10"); + + Assert.False(BanExemptions.IsExempt(_listed, BanReasons.Blocklist, NeverCalled)); + Assert.False(BanExemptions.IsExempt(_listed, "some-future-reason", NeverCalled)); + } + + [Fact] + public void File_allowlist_does_not_spend_the_earned_lists_strikes() + { + WithFileAllowlist("192.0.2.10"); + + // Unconditional, so the revocable list must not be consulted -- that would burn a strike. + Assert.True(BanExemptions.IsExempt(_listed, BanReasons.RateLimit, NeverCalled)); + } + + [Fact] + public void Falls_through_to_the_login_allowlist_when_not_file_listed() + { + WithEmptyFileAllowlist(); + + var consulted = 0; + + var result = BanExemptions.IsExempt( + _unlisted, + BanReasons.RateLimit, + (_, _) => + { + consulted++; + return true; + } + ); + + Assert.True(result); + Assert.Equal(1, consulted); + } + + [Fact] + public void Null_address_is_never_exempt() + { + WithEmptyFileAllowlist(); + + Assert.False(BanExemptions.IsExempt(null, BanReasons.RateLimit, NeverCalled)); + } + + private static bool NeverCalled(IPAddress address, string reason) + { + Assert.Fail("The login allowlist must not be consulted once the answer is already decided."); + return false; + } + + private static bool AlwaysFalse(IPAddress address, string reason) => false; +} diff --git a/Projects/UOContent.Tests/Tests/Network/LoginAllowlist/LoginAllowlistTests.cs b/Projects/UOContent.Tests/Tests/Network/LoginAllowlist/LoginAllowlistTests.cs new file mode 100644 index 000000000..7f3780e58 --- /dev/null +++ b/Projects/UOContent.Tests/Tests/Network/LoginAllowlist/LoginAllowlistTests.cs @@ -0,0 +1,218 @@ +/************************************************************************* + * ModernUO * + * Copyright 2019-2026 - ModernUO Development Team * + * Email: hi@modernuo.com * + * File: LoginAllowlistTests.cs * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +using System.Net; +using Server.Network; +using Server.Network.Bans; +using Xunit; + +namespace Server.Tests.Network.LoginAllowlists; + +// The list is static, so every test resets it first. Addresses come from TEST-NET-3 (203.0.113.0/24) so they +// cannot collide with the blocklist tests if xUnit runs the two classes at the same time. +public class LoginAllowlistTests +{ + private const long Ttl = 90 * 24 * 60 * 60; // 90 days, the shipped default + private const long Window = 3600; + private const long Now = 1_800_000_000; + + private static void Reset(bool enabled = true, int strikes = 0) => + LoginAllowlist.LoadForTesting(enabled, Ttl, strikes, Window); + + [Fact] + public void Recent_login_allows_its_address() + { + Reset(); + var ip = IPAddress.Parse("203.0.113.10"); + + LoginAllowlist.RecordLogin(ip, Now); + + Assert.True(LoginAllowlist.IsAllowed(ip, Now)); + Assert.True(LoginAllowlist.IsAllowed(ip, Now + Ttl / 2)); + } + + [Fact] + public void Entry_lapses_once_past_the_ttl() + { + Reset(); + var ip = IPAddress.Parse("203.0.113.11"); + + LoginAllowlist.RecordLogin(ip, Now); + + Assert.True(LoginAllowlist.IsAllowed(ip, Now + Ttl)); // the boundary still counts + Assert.False(LoginAllowlist.IsAllowed(ip, Now + Ttl + 1)); // a second later it does not + } + + [Fact] + public void Logging_in_again_renews_the_window() + { + Reset(); + var ip = IPAddress.Parse("203.0.113.12"); + + LoginAllowlist.RecordLogin(ip, Now); + LoginAllowlist.RecordLogin(ip, Now + Ttl); // still allowed here, so the entry is refreshed + + Assert.True(LoginAllowlist.IsAllowed(ip, Now + Ttl + Ttl)); + } + + [Fact] + public void Unknown_address_is_not_allowed() + { + Reset(); + LoginAllowlist.RecordLogin(IPAddress.Parse("203.0.113.13"), Now); + + Assert.False(LoginAllowlist.IsAllowed(IPAddress.Parse("203.0.113.14"), Now)); + } + + [Fact] + public void Private_addresses_are_never_recorded() + { + Reset(); + + // A LAN or loopback login says nothing about the public internet. + LoginAllowlist.RecordLogin(IPAddress.Parse("10.0.0.5"), Now); + LoginAllowlist.RecordLogin(IPAddress.Loopback, Now); + + Assert.False(LoginAllowlist.IsAllowed(IPAddress.Parse("10.0.0.5"), Now)); + Assert.False(LoginAllowlist.IsAllowed(IPAddress.Loopback, Now)); + Assert.Equal(0, LoginAllowlist.Count); + } + + [Fact] + public void Disabled_list_allows_nobody() + { + Reset(enabled: false); + var ip = IPAddress.Parse("203.0.113.15"); + + LoginAllowlist.RecordLogin(ip, Now); + + Assert.False(LoginAllowlist.IsAllowed(ip, Now)); + Assert.Equal(0, LoginAllowlist.Count); + } + + [Fact] + public void Null_address_is_handled() + { + Reset(); + + LoginAllowlist.RecordLogin(null, Now); + + Assert.False(LoginAllowlist.IsAllowed(null, Now)); + } + + // ----- escalation ------------------------------------------------------------------------------- + + [Fact] + public void Manual_bans_are_never_exempt() + { + Reset(); + var ip = IPAddress.Parse("203.0.113.20"); + LoginAllowlist.RecordLogin(ip, Now); + + // An explicit decision must reach the reporters even for an allowlisted address. + Assert.False(LoginAllowlist.IsExemptFromEscalation(ip, BanReasons.Manual, Now)); + } + + [Fact] + public void Unopted_reasons_are_never_exempt() + { + Reset(); + var ip = IPAddress.Parse("203.0.113.21"); + LoginAllowlist.RecordLogin(ip, Now); + + // A reason nobody opted into IsBehavioral escalates normally rather than inheriting an exemption. + Assert.False(LoginAllowlist.IsExemptFromEscalation(ip, "some-future-reason", Now)); + Assert.False(LoginAllowlist.IsExemptFromEscalation(ip, BanReasons.Blocklist, Now)); + } + + [Fact] + public void Behavioral_reasons_are_exempt_while_allowed() + { + Reset(); + var ip = IPAddress.Parse("203.0.113.22"); + LoginAllowlist.RecordLogin(ip, Now); + + Assert.True(LoginAllowlist.IsExemptFromEscalation(ip, BanReasons.RateLimit, Now)); + Assert.True(LoginAllowlist.IsExemptFromEscalation(ip, BanReasons.SilentConnect, Now)); + Assert.True(LoginAllowlist.IsExemptFromEscalation(ip, BanReasons.InvalidSeed, Now)); + } + + [Fact] + public void Entry_is_revoked_once_the_strikes_run_out() + { + Reset(strikes: 3); + var ip = IPAddress.Parse("203.0.113.23"); + LoginAllowlist.RecordLogin(ip, Now); + + Assert.True(LoginAllowlist.IsExemptFromEscalation(ip, BanReasons.RateLimit, Now)); + Assert.True(LoginAllowlist.IsExemptFromEscalation(ip, BanReasons.RateLimit, Now)); + + // The third strike is the one that escalates, and it takes the entry with it. + Assert.False(LoginAllowlist.IsExemptFromEscalation(ip, BanReasons.RateLimit, Now)); + Assert.False(LoginAllowlist.IsAllowed(ip, Now)); + + // Still revoked afterwards, so everything from here escalates too. + Assert.False(LoginAllowlist.IsExemptFromEscalation(ip, BanReasons.RateLimit, Now)); + } + + [Fact] + public void Quiet_window_clears_the_tally() + { + Reset(strikes: 3); + var ip = IPAddress.Parse("203.0.113.24"); + LoginAllowlist.RecordLogin(ip, Now); + + Assert.True(LoginAllowlist.IsExemptFromEscalation(ip, BanReasons.RateLimit, Now)); + Assert.True(LoginAllowlist.IsExemptFromEscalation(ip, BanReasons.RateLimit, Now)); + + // Past the window the count restarts, so an occasional tripper never accumulates to revocation. + var later = Now + Window + 1; + Assert.True(LoginAllowlist.IsExemptFromEscalation(ip, BanReasons.RateLimit, later)); + Assert.True(LoginAllowlist.IsExemptFromEscalation(ip, BanReasons.RateLimit, later)); + Assert.True(LoginAllowlist.IsAllowed(ip, later)); + } + + [Fact] + public void Logging_in_again_forgives_accumulated_strikes() + { + Reset(strikes: 3); + var ip = IPAddress.Parse("203.0.113.25"); + LoginAllowlist.RecordLogin(ip, Now); + + Assert.True(LoginAllowlist.IsExemptFromEscalation(ip, BanReasons.RateLimit, Now)); + Assert.True(LoginAllowlist.IsExemptFromEscalation(ip, BanReasons.RateLimit, Now)); + + LoginAllowlist.RecordLogin(ip, Now); // someone proved they hold an account again + + Assert.True(LoginAllowlist.IsExemptFromEscalation(ip, BanReasons.RateLimit, Now)); + Assert.True(LoginAllowlist.IsExemptFromEscalation(ip, BanReasons.RateLimit, Now)); + Assert.True(LoginAllowlist.IsAllowed(ip, Now)); + } + + [Fact] + public void Zero_threshold_disables_revocation() + { + Reset(strikes: 0); + var ip = IPAddress.Parse("203.0.113.26"); + LoginAllowlist.RecordLogin(ip, Now); + + for (var i = 0; i < 50; i++) + { + Assert.True(LoginAllowlist.IsExemptFromEscalation(ip, BanReasons.RateLimit, Now)); + } + + Assert.True(LoginAllowlist.IsAllowed(ip, Now)); + } +} diff --git a/Projects/UOContent/Accounting/AccountHandler.cs b/Projects/UOContent/Accounting/AccountHandler.cs index 1e676ed9e..ea0c4b9e0 100644 --- a/Projects/UOContent/Accounting/AccountHandler.cs +++ b/Projects/UOContent/Accounting/AccountHandler.cs @@ -324,6 +324,7 @@ public static class AccountHandler e.Accepted = true; acct.LogAccess(e.State); + LoginAllowlist.RecordLogin(e.State?.Address); } } @@ -355,6 +356,7 @@ public static class AccountHandler else { acct.LogAccess(e.State); + LoginAllowlist.RecordLogin(e.State?.Address); logger.Information("Login: {NetState} Account '{Username}' at character list", e.State, un); e.State.Account = acct; diff --git a/Projects/UOContent/Commands/Generic/Commands/Commands.cs b/Projects/UOContent/Commands/Generic/Commands/Commands.cs index b3fa95350..102d5a2aa 100644 --- a/Projects/UOContent/Commands/Generic/Commands/Commands.cs +++ b/Projects/UOContent/Commands/Generic/Commands/Commands.cs @@ -1156,7 +1156,7 @@ namespace Server.Commands.Generic try { Firewall.Add(new SingleIpFirewallEntry(state.Address)); - BanChannel.Report(state.Address, TimeSpan.Zero, "manual"); + BanChannel.Report(state.Address, TimeSpan.Zero, BanReasons.Manual); AddResponse("They have been firewalled."); } catch (Exception ex) diff --git a/Projects/UOContent/Gumps/AdminGump.cs b/Projects/UOContent/Gumps/AdminGump.cs index 1e788372e..1fe917b55 100644 --- a/Projects/UOContent/Gumps/AdminGump.cs +++ b/Projects/UOContent/Gumps/AdminGump.cs @@ -1745,7 +1745,7 @@ namespace Server.Gumps for (var i = 0; i < a.LoginIPs.Length; ++i) { Firewall.Add(new SingleIpFirewallEntry(a.LoginIPs[i])); - BanChannel.Report(a.LoginIPs[i], TimeSpan.Zero, "manual"); + BanChannel.Report(a.LoginIPs[i], TimeSpan.Zero, BanReasons.Manual); } notice = "All addresses in the list have been firewalled."; @@ -1774,7 +1774,7 @@ namespace Server.Gumps if (firewallEntry.MinIpAddress == firewallEntry.MaxIpAddress) { - BanChannel.Report(firewallEntry.MinIpAddress.ToIpAddress(), TimeSpan.Zero, "manual"); + BanChannel.Report(firewallEntry.MinIpAddress.ToIpAddress(), TimeSpan.Zero, BanReasons.Manual); } notice = $"{toFirewall} : Added to firewall."; @@ -3596,7 +3596,7 @@ namespace Server.Gumps BanChannel.Report( firewallEntry.MinIpAddress.ToIpAddress(), TimeSpan.Zero, - "manual" + BanReasons.Manual ); } diff --git a/Projects/UOContent/Network/AutoDenylist/AutoDenylist.cs b/Projects/UOContent/Network/AutoDenylist/AutoDenylist.cs new file mode 100644 index 000000000..a55280e1a --- /dev/null +++ b/Projects/UOContent/Network/AutoDenylist/AutoDenylist.cs @@ -0,0 +1,219 @@ +/************************************************************************* + * ModernUO * + * Copyright 2019-2026 - ModernUO Development Team * + * Email: hi@modernuo.com * + * File: AutoDenylist.cs * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +using System; +using System.Collections.Generic; +using System.Net; +using System.Threading; +using Server.Collections; +using Server.Logging; +using Server.Network.Bans; + +namespace Server.Network; + +/// +/// A short-lived, in-memory denylist of addresses the shard itself just caught misbehaving. +/// +/// +/// The local half of promotion. Contributing to CrowdSec only helps once an OS bouncer reacts; until then +/// every reconnect costs a socket, a buffer and a NetState slot — and the verdicts that matter most +/// are reachable only after reading bytes, like a zero seed. It is also the whole defence on a shard running +/// no bouncer, which is the default. Not persisted, by design: a holding pen that survives restarts is a ban +/// without a ban's review. Only verdicts are held. +/// +public static class AutoDenylist +{ + private static readonly ILogger logger = LogFactory.GetLogger(typeof(AutoDenylist)); + + // Address (normalized v6 bits) -> Core.TickCount at which the hold lapses. Loop-only. + private static readonly Dictionary _held = []; + + private static bool _enabled; + private static long _durationMs; + private static int _maxEntries; + private static bool _warnedFull; + + public static int Count => _held.Count; + + public static void Configure() + { + AutoDenylistConfiguration.Load(); + var s = AutoDenylistConfiguration.Settings; + + _enabled = s.Enabled && s.Duration > TimeSpan.Zero && s.MaxEntries > 0; + if (!_enabled) + { + return; + } + + _durationMs = (long)s.Duration.TotalMilliseconds; + _maxEntries = s.MaxEntries; + + ConnectionFilters.Register(new AutoDenylistFilter()); + BanChannel.Register(new AutoDenylistReporter()); + } + + /// + /// Holds an address for the configured duration. Ignores non-behavioural verdicts and refuses to grow + /// past the cap: the flood this exists for must not become a memory leak. + /// + public static void Hold(IPAddress address, string reason) => Hold(address, reason, Core.TickCount); + + internal static bool Hold(IPAddress address, string reason, long nowTicks) + { + if (!_enabled || address == null || !BanReasons.IsBehavioral(reason)) + { + return false; + } + + var key = address.ToUInt128(); + + // An address already held is just extended, so no cap check is needed. + if (!_held.ContainsKey(key) && _held.Count >= _maxEntries) + { + Sweep(nowTicks); + + if (_held.Count >= _maxEntries) + { + if (!_warnedFull) + { + _warnedFull = true; + logger.Warning( + "Auto-denylist is full at {Max} addresses; further detections are disconnected but not held", + _maxEntries + ); + } + + return false; + } + } + + _held[key] = nowTicks + _durationMs; + return true; + } + + public static bool IsDenied(IPAddress address) => IsDenied(address, Core.TickCount); + + /// The pure decision, split out so the accept-path policy can be tested without a clock. + internal static bool IsDenied(IPAddress address, long nowTicks) + { + if (!_enabled || address == null) + { + return false; + } + + // Decided on read, so a lapsed hold cannot deny even before the sweep. Subtraction: TickCount wraps. + return _held.TryGetValue(address.ToUInt128(), out var expires) && expires - nowTicks > 0; + } + + /// Releases an address early, e.g. when an operator retracts a ban. + public static void Release(IPAddress address) + { + if (_enabled && address != null) + { + _held.Remove(address.ToUInt128()); + } + } + + internal static void Sweep(long nowTicks) + { + if (_held.Count == 0) + { + return; + } + + using var lapsed = new PooledRefList(16); + + foreach (var (address, expires) in _held) + { + if (expires - nowTicks <= 0) + { + lapsed.Add(address); + } + } + + for (var i = 0; i < lapsed.Count; i++) + { + _held.Remove(lapsed[i]); + } + + if (lapsed.Count > 0) + { + _warnedFull = false; + } + } + + internal static void LoadForTesting(bool enabled, long durationMs, int maxEntries) + { + _held.Clear(); + _enabled = enabled; + _durationMs = durationMs; + _maxEntries = maxEntries; + _warnedFull = false; + } +} + +/// Accept-path gate for . +public sealed class AutoDenylistFilter : IConnectionFilter +{ + public string Name => "auto-denylist"; + + public void Register() + { + } + + public void Start(CancellationToken token) + { + // Only an optimisation: IsDenied expires on read. + Timer.DelayCall(TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1), () => AutoDenylist.Sweep(Core.TickCount)); + } + + public void Stop() + { + } + + public bool ShouldDeny(IPAddress address) => AutoDenylist.IsDenied(address); +} + +/// +/// Feeds from the ban channel. A reporter rather than a direct call, because the +/// detection sites live in the engine and must not reach into content. +/// +public sealed class AutoDenylistReporter : IBanReporter +{ + public string Name => "auto-denylist"; + + public bool CanRetract => true; + + public void Register() + { + } + + public void Start(CancellationToken token) + { + } + + public void Stop() + { + } + + /// + /// The contributed is ignored: how long a bouncer should ban an address is a + /// different question from how long this shard holds it at accept. + /// + public void Report(IPAddress address, TimeSpan ttl, string reason) => AutoDenylist.Hold(address, reason); + + public void Retract(IPAddress address) => AutoDenylist.Release(address); +} diff --git a/Projects/UOContent/Network/AutoDenylist/AutoDenylistConfiguration.cs b/Projects/UOContent/Network/AutoDenylist/AutoDenylistConfiguration.cs new file mode 100644 index 000000000..4d45da350 --- /dev/null +++ b/Projects/UOContent/Network/AutoDenylist/AutoDenylistConfiguration.cs @@ -0,0 +1,81 @@ +/************************************************************************* + * ModernUO * + * Copyright 2019-2026 - ModernUO Development Team * + * Email: hi@modernuo.com * + * File: AutoDenylistConfiguration.cs * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +using System; +using System.IO; +using System.Text.Json.Serialization; +using Server.Json; + +namespace Server.Network; + +/// +/// Loads the from Configuration/auto-denylist.json (matching the +/// per-feature JSON config pattern used by BlocklistConfiguration). Loaded once; a missing file writes +/// a template so operators have something to edit. +/// +public static class AutoDenylistConfiguration +{ + private const string _path = "Configuration/auto-denylist.json"; + + public static AutoDenylistSettings Settings { get; private set; } + + public static void Load() + { + var path = Path.Join(Core.BaseDirectory, _path); + + if (File.Exists(path)) + { + Settings = JsonConfig.Deserialize(path); + } + else + { + Settings = new AutoDenylistSettings(); + Save(); + } + } + + private static void Save() + { + JsonConfig.Serialize(Path.Join(Core.BaseDirectory, _path), Settings); + } +} + +/// Bound configuration for . +public record AutoDenylistSettings +{ + /// Whether behavioural detections are held locally. Disabled makes the filter inert. + [JsonPropertyName("enabled")] + public bool Enabled { get; set; } = true; + + /// + /// How long an address is denied at accept after the shard catches it misbehaving. Deliberately + /// independent of the duration reported to external bouncers: this is a local holding pen, not a ban. + /// + /// + /// Short on purpose: it covers the gap before an OS bouncer reacts, and blunts a flood on shards running + /// none. An address still attacking is simply re-detected and re-added, so the list sustains itself while + /// a mistake clears on its own. + /// + [JsonPropertyName("duration")] + public TimeSpan Duration { get; set; } = TimeSpan.FromMinutes(15); + + /// + /// Hard cap on tracked addresses: a distinct-source flood is the case this exists for, so the cap is what + /// stops it becoming the exhaustion it prevents. At the cap new addresses are not tracked, but are still + /// disconnected by whichever gate detected them. + /// + [JsonPropertyName("maxEntries")] + public int MaxEntries { get; set; } = 65536; +} diff --git a/Projects/UOContent/Network/BanExemptions.cs b/Projects/UOContent/Network/BanExemptions.cs new file mode 100644 index 000000000..5566bd5e7 --- /dev/null +++ b/Projects/UOContent/Network/BanExemptions.cs @@ -0,0 +1,61 @@ +/************************************************************************* + * ModernUO * + * Copyright 2019-2026 - ModernUO Development Team * + * Email: hi@modernuo.com * + * File: BanExemptions.cs * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +using System; +using System.Net; +using Server.Network.Bans; + +namespace Server.Network; + +/// +/// Combines and into the one answer +/// asks for, so neither source has to know about the other. +/// +public static class BanExemptions +{ + public static void Configure() + { + BanChannel.IsExempt = IsExempt; + } + + public static bool IsExempt(IPAddress address, string reason) => + IsExempt(address, reason, LoginAllowlist.IsExemptFromEscalation); + + /// + /// Split for testing. is stateful — calling it spends a strike — so it + /// must not be invoked once the answer is already decided. + /// + internal static bool IsExempt(IPAddress address, string reason, Func loginAllowlist) + { + if (address == null) + { + return false; + } + + // Never suppress an operator's explicit ban. Checked first so it also costs no strike. + if (!BanReasons.IsBehavioral(reason)) + { + return false; + } + + // Deliberate and unconditional, so it wins and must not spend the earned list's strikes. + if (FileAllowlist.Contains(address)) + { + return true; + } + + return loginAllowlist(address, reason); + } +} diff --git a/Projects/UOContent/Misc/Blocklist/BlocklistConfiguration.cs b/Projects/UOContent/Network/Blocklist/BlocklistConfiguration.cs similarity index 85% rename from Projects/UOContent/Misc/Blocklist/BlocklistConfiguration.cs rename to Projects/UOContent/Network/Blocklist/BlocklistConfiguration.cs index ca39bd53e..8a2df5ca6 100644 --- a/Projects/UOContent/Misc/Blocklist/BlocklistConfiguration.cs +++ b/Projects/UOContent/Network/Blocklist/BlocklistConfiguration.cs @@ -67,6 +67,19 @@ public record BlocklistSettings [JsonPropertyName("file")] public string File { get; set; } = "Configuration/ip-blocklist.txt"; + /// + /// Addresses that must never be blocked and never escalated, in the blocklist's own format. The same + /// files tools/Export-IpBlocklist.ps1 subtracts at generation time; the shard reads them so an + /// entry also suppresses ban contributions, which the generator alone cannot do. See + /// . + /// + /// + /// The filename may contain wildcards, which is how the default picks up a carve-out an admin adds + /// without anyone editing this file. + /// + [JsonPropertyName("allowlistFiles")] + public string[] AllowlistFiles { get; set; } = ["Configuration/ip-allowlist*.txt"]; + /// How often the file is checked for changes. Reloads only happen when it actually changed. [JsonPropertyName("reloadInterval")] public TimeSpan ReloadInterval { get; set; } = TimeSpan.FromSeconds(60); diff --git a/Projects/UOContent/Misc/Blocklist/BlocklistFile.cs b/Projects/UOContent/Network/Blocklist/BlocklistFile.cs similarity index 100% rename from Projects/UOContent/Misc/Blocklist/BlocklistFile.cs rename to Projects/UOContent/Network/Blocklist/BlocklistFile.cs diff --git a/Projects/UOContent/Misc/Blocklist/BlocklistFilter.cs b/Projects/UOContent/Network/Blocklist/BlocklistFilter.cs similarity index 92% rename from Projects/UOContent/Misc/Blocklist/BlocklistFilter.cs rename to Projects/UOContent/Network/Blocklist/BlocklistFilter.cs index aeca58477..36354160b 100644 --- a/Projects/UOContent/Misc/Blocklist/BlocklistFilter.cs +++ b/Projects/UOContent/Network/Blocklist/BlocklistFilter.cs @@ -133,7 +133,7 @@ public sealed class BlocklistFilter : IConnectionFilter if (shouldReport) { // Demand-page this address up to the OS-level bouncer. Enqueue-only; never blocks the loop. - BanChannel.Report(address, _banDuration, "blocklist"); + BanChannel.Report(address, _banDuration, BanReasons.Blocklist); } return true; @@ -152,6 +152,21 @@ public sealed class BlocklistFilter : IConnectionFilter return false; } + // Both are asked only once the list has matched, so they cost the common accept nothing. The file + // list is usually redundant because the generator subtracts it — except right after an operator adds + // an entry without regenerating, which is exactly when someone is waiting to get back in. + if (FileAllowlist.Contains(address)) + { + return false; + } + + // A feed listing an address a real player logged in from recently is far more often a false positive + // than a compromise. + if (LoginAllowlist.IsAllowed(address)) + { + return false; + } + if (_reportHits) { shouldReport = _guard.TryMark(address.ToUInt128(), nowTicks, _suppressionMs); diff --git a/Projects/UOContent/Misc/Blocklist/BlocklistSnapshot.cs b/Projects/UOContent/Network/Blocklist/BlocklistSnapshot.cs similarity index 96% rename from Projects/UOContent/Misc/Blocklist/BlocklistSnapshot.cs rename to Projects/UOContent/Network/Blocklist/BlocklistSnapshot.cs index 52839512c..29b2e5176 100644 --- a/Projects/UOContent/Misc/Blocklist/BlocklistSnapshot.cs +++ b/Projects/UOContent/Network/Blocklist/BlocklistSnapshot.cs @@ -182,6 +182,12 @@ public sealed class BlocklistSnapshot return false; } + /// + /// Plain set membership, for callers whose set is an ALLOWlist (see ) and for + /// whom would read backwards. The interval machinery is direction-agnostic. + /// + public bool Contains(IPAddress ip) => IsBanned(ip); + public bool IsBanned(IPAddress ip) { if (ip.IsIPv4MappedToIPv6) diff --git a/Projects/UOContent/Network/Blocklist/FileAllowlist.cs b/Projects/UOContent/Network/Blocklist/FileAllowlist.cs new file mode 100644 index 000000000..0d5582713 --- /dev/null +++ b/Projects/UOContent/Network/Blocklist/FileAllowlist.cs @@ -0,0 +1,313 @@ +/************************************************************************* + * ModernUO * + * Copyright 2019-2026 - ModernUO Development Team * + * Email: hi@modernuo.com * + * File: FileAllowlist.cs * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Server.Logging; + +namespace Server.Network.Bans; + +/// +/// The operator's own "leave this address alone" list, read from the same files +/// tools/Export-IpBlocklist.ps1 subtracts at generation time. +/// +/// +/// The generator already subtracts these from the blocklist, but that only covers being BLOCKED. +/// Behavioural detections never consult the blocklist, so without reading the files here a carve-out is +/// quietly routed around: one scanner behind a shared CGNAT address is enough to get the whole address +/// contributed and firewalled. Reading them also means an entry applies on the next reload rather than the +/// next regeneration. Unconditional, unlike , but still no shield against a +/// manual ban — see . +/// +public static class FileAllowlist +{ + private static readonly ILogger logger = LogFactory.GetLogger(typeof(FileAllowlist)); + + // Written by the reload poll (off-loop), read by the accept path (game loop): a single volatile + // reference swap is the whole synchronization story — readers see the old or the new snapshot, whole. + private static volatile BlocklistSnapshot _snapshot = BlocklistSnapshot.Empty; + + private static string[] _patterns = []; + private static TimeSpan _interval = TimeSpan.FromSeconds(60); + private static long _lastStamp; + private static CancellationTokenSource _cts; + + public static int Count => _snapshot.Count; + + /// True when an operator listed this address. Safe before . + public static bool Contains(IPAddress address) => address != null && _snapshot.Contains(address); + + public static void Initialize() + { + // BlocklistFilter.Register ran during the Configure sweep, so the settings are populated. + var settings = BlocklistConfiguration.Settings; + if (settings == null) + { + return; + } + + _patterns = ResolvePaths(settings.AllowlistFiles); + _interval = settings.ReloadInterval <= TimeSpan.Zero ? TimeSpan.FromSeconds(60) : settings.ReloadInterval; + + if (_patterns.Length == 0) + { + logger.Information("File allowlist disabled (\"allowlistFiles\" empty in blocklist.json)"); + return; + } + + Reload(); + + _cts = CancellationTokenSource.CreateLinkedTokenSource(Core.ClosingTokenSource.Token); + _ = Task.Run(() => PollLoop(_cts.Token), _cts.Token); + } + + public static void Stop() + { + _cts?.Cancel(); + _cts?.Dispose(); + _cts = null; + } + + private static string[] ResolvePaths(string[] configured) + { + if (configured == null) + { + return []; + } + + var resolved = new string[configured.Length]; + var count = 0; + + for (var i = 0; i < configured.Length; i++) + { + var path = configured[i]; + if (string.IsNullOrWhiteSpace(path)) + { + continue; + } + + // Relative resolves against BaseDirectory, never the working directory, which differs when the + // shard is launched from elsewhere. Absolute is as-is, so shards can share a list. + resolved[count++] = Path.IsPathRooted(path) ? path : Path.Join(Core.BaseDirectory, path); + } + + Array.Resize(ref resolved, count); + return resolved; + } + + /// + /// Expands the configured patterns to actual files. Done per poll rather than once, so a carve-out an + /// admin adds is picked up without a restart. + /// + private static string[] ExpandPaths() + { + var files = new List(); + + for (var i = 0; i < _patterns.Length; i++) + { + var pattern = _patterns[i]; + var name = Path.GetFileName(pattern); + + if (name.IndexOf('*') < 0 && name.IndexOf('?') < 0) + { + if (File.Exists(pattern)) + { + files.Add(pattern); + } + + continue; + } + + try + { + var dir = Path.GetDirectoryName(pattern); + if (string.IsNullOrEmpty(dir) || !Directory.Exists(dir)) + { + continue; + } + + var matches = Directory.GetFiles(dir, name); + Array.Sort(matches, StringComparer.Ordinal); + + for (var j = 0; j < matches.Length; j++) + { + // Windows wildcard matching still honours legacy short names, so ".txt" can pull in the + // generator's ".txt.tmp" mid-swap. Check the real extension. + if (matches[j].EndsWith(".txt", StringComparison.OrdinalIgnoreCase)) + { + files.Add(matches[j]); + } + } + } + catch + { + // Unreadable directory; the next poll retries. + } + } + + return files.ToArray(); + } + + private static async ValueTask PollLoop(CancellationToken token) + { + while (!token.IsCancellationRequested) + { + try + { + await Task.Delay(_interval, token); + } + catch (OperationCanceledException) + { + return; + } + + try + { + if (Stamp() != _lastStamp) + { + // A save owns the disk and nothing here is urgent. + // See the threading policy in CLAUDE.md (rules #3 and #10). + while (World.Saving || World.WorldState == WorldState.PendingSave) + { + await Task.Delay(TimeSpan.FromSeconds(1), token); + } + + Reload(); + } + } + catch (OperationCanceledException) + { + return; + } + catch (Exception e) + { + logger.Warning(e, "File allowlist reload check failed; keeping last snapshot ({Count})", Count); + } + } + } + + /// + /// Change fingerprint across every configured file. A missing file contributes nothing, so creating or + /// deleting one also registers as a change. + /// + private static long Stamp() + { + var stamp = 0L; + var paths = ExpandPaths(); + + for (var i = 0; i < paths.Length; i++) + { + try + { + var info = new FileInfo(paths[i]); + if (info.Exists) + { + stamp = stamp * 31 + info.LastWriteTimeUtc.Ticks + info.Length; + } + } + catch + { + // Mid-swap by the generator; the next poll picks it up. + } + } + + return stamp; + } + + private static void Reload() + { + // Fingerprint BEFORE parsing, so it describes the version being read. Capturing after could skip a + // version; a stale fingerprint only costs an extra reload. + var stamp = Stamp(); + var combined = ReadAll(out var files); + + // Reuses the blocklist parser and interval index: an address set is direction-agnostic. + var next = combined.Length == 0 + ? BlocklistSnapshot.Empty + : BlocklistSnapshot.Build(combined, out _, out _); + + _snapshot = next; // single volatile swap; readers see old or new whole + _lastStamp = stamp; + + logger.Information( + "File allowlist loaded {Count} range(s) from {Files} file(s)", + next.Count, + files + ); + } + + /// + /// Concatenates every configured file into one buffer. The parser is line-based, so a newline join is + /// enough, and membership stays a single lookup. + /// + private static byte[] ReadAll(out int files) + { + files = 0; + + var paths = ExpandPaths(); + var chunks = new byte[paths.Length][]; + var total = 0; + + for (var i = 0; i < paths.Length; i++) + { + try + { + if (!File.Exists(paths[i])) + { + continue; + } + + var bytes = File.ReadAllBytes(paths[i]); + chunks[i] = bytes; + total += bytes.Length + 1; // + newline separator + files++; + } + catch (Exception e) + { + // Fail open per file: losing one entry beats refusing to load the rest. + logger.Warning(e, "Could not read allowlist \"{Path}\"", paths[i]); + } + } + + if (total == 0) + { + return []; + } + + var combined = new byte[total]; + var offset = 0; + + for (var i = 0; i < chunks.Length; i++) + { + var chunk = chunks[i]; + if (chunk == null) + { + continue; + } + + Buffer.BlockCopy(chunk, 0, combined, offset, chunk.Length); + offset += chunk.Length; + combined[offset++] = (byte)'\n'; + } + + return combined; + } + + internal static void LoadForTesting(BlocklistSnapshot snapshot) => _snapshot = snapshot ?? BlocklistSnapshot.Empty; +} diff --git a/Projects/UOContent/Misc/Blocklist/PromotedGuard.cs b/Projects/UOContent/Network/Blocklist/PromotedGuard.cs similarity index 100% rename from Projects/UOContent/Misc/Blocklist/PromotedGuard.cs rename to Projects/UOContent/Network/Blocklist/PromotedGuard.cs diff --git a/Projects/UOContent/Misc/CrowdSec/CrowdSecAlert.cs b/Projects/UOContent/Network/CrowdSec/CrowdSecAlert.cs similarity index 100% rename from Projects/UOContent/Misc/CrowdSec/CrowdSecAlert.cs rename to Projects/UOContent/Network/CrowdSec/CrowdSecAlert.cs diff --git a/Projects/UOContent/Misc/CrowdSec/CrowdSecAlertClient.cs b/Projects/UOContent/Network/CrowdSec/CrowdSecAlertClient.cs similarity index 100% rename from Projects/UOContent/Misc/CrowdSec/CrowdSecAlertClient.cs rename to Projects/UOContent/Network/CrowdSec/CrowdSecAlertClient.cs diff --git a/Projects/UOContent/Misc/CrowdSec/CrowdSecConfiguration.cs b/Projects/UOContent/Network/CrowdSec/CrowdSecConfiguration.cs similarity index 100% rename from Projects/UOContent/Misc/CrowdSec/CrowdSecConfiguration.cs rename to Projects/UOContent/Network/CrowdSec/CrowdSecConfiguration.cs diff --git a/Projects/UOContent/Misc/CrowdSec/CrowdSecReporter.cs b/Projects/UOContent/Network/CrowdSec/CrowdSecReporter.cs similarity index 99% rename from Projects/UOContent/Misc/CrowdSec/CrowdSecReporter.cs rename to Projects/UOContent/Network/CrowdSec/CrowdSecReporter.cs index e38aa17a6..385534d2a 100644 --- a/Projects/UOContent/Misc/CrowdSec/CrowdSecReporter.cs +++ b/Projects/UOContent/Network/CrowdSec/CrowdSecReporter.cs @@ -363,7 +363,7 @@ public sealed class CrowdSecReporter : IBanReporter foreach (var (value, item) in byIp) { - var ttl = item.Reason == "manual" || item.Ttl <= TimeSpan.Zero ? settings.ManualBanDuration : item.Ttl; + var ttl = item.Reason == BanReasons.Manual || item.Ttl <= TimeSpan.Zero ? settings.ManualBanDuration : item.Ttl; var scenario = $"{settings.Origin}/{item.Reason}"; alerts.Add(new CrowdSecAlert diff --git a/Projects/UOContent/Misc/Firewall/BaseFirewallEntry.cs b/Projects/UOContent/Network/Firewall/BaseFirewallEntry.cs similarity index 100% rename from Projects/UOContent/Misc/Firewall/BaseFirewallEntry.cs rename to Projects/UOContent/Network/Firewall/BaseFirewallEntry.cs diff --git a/Projects/UOContent/Misc/Firewall/CidrFirewallEntry.cs b/Projects/UOContent/Network/Firewall/CidrFirewallEntry.cs similarity index 100% rename from Projects/UOContent/Misc/Firewall/CidrFirewallEntry.cs rename to Projects/UOContent/Network/Firewall/CidrFirewallEntry.cs diff --git a/Projects/UOContent/Misc/Firewall/Firewall.cs b/Projects/UOContent/Network/Firewall/Firewall.cs similarity index 100% rename from Projects/UOContent/Misc/Firewall/Firewall.cs rename to Projects/UOContent/Network/Firewall/Firewall.cs diff --git a/Projects/UOContent/Misc/Firewall/FirewallConnectionFilter.cs b/Projects/UOContent/Network/Firewall/FirewallConnectionFilter.cs similarity index 100% rename from Projects/UOContent/Misc/Firewall/FirewallConnectionFilter.cs rename to Projects/UOContent/Network/Firewall/FirewallConnectionFilter.cs diff --git a/Projects/UOContent/Misc/Firewall/FirewallSettings.cs b/Projects/UOContent/Network/Firewall/FirewallSettings.cs similarity index 100% rename from Projects/UOContent/Misc/Firewall/FirewallSettings.cs rename to Projects/UOContent/Network/Firewall/FirewallSettings.cs diff --git a/Projects/UOContent/Misc/Firewall/IFirewallEntry.cs b/Projects/UOContent/Network/Firewall/IFirewallEntry.cs similarity index 100% rename from Projects/UOContent/Misc/Firewall/IFirewallEntry.cs rename to Projects/UOContent/Network/Firewall/IFirewallEntry.cs diff --git a/Projects/UOContent/Misc/Firewall/SingleIpFirewallEntry.cs b/Projects/UOContent/Network/Firewall/SingleIpFirewallEntry.cs similarity index 100% rename from Projects/UOContent/Misc/Firewall/SingleIpFirewallEntry.cs rename to Projects/UOContent/Network/Firewall/SingleIpFirewallEntry.cs diff --git a/Projects/UOContent/Network/LoginAllowlist/LoginAllowlist.cs b/Projects/UOContent/Network/LoginAllowlist/LoginAllowlist.cs new file mode 100644 index 000000000..ed65b5ec6 --- /dev/null +++ b/Projects/UOContent/Network/LoginAllowlist/LoginAllowlist.cs @@ -0,0 +1,385 @@ +/************************************************************************* + * ModernUO * + * Copyright 2019-2026 - ModernUO Development Team * + * Email: hi@modernuo.com * + * File: LoginAllowlist.cs * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Net; +using System.Text; +using System.Threading.Tasks; +using Server.Collections; +using Server.Logging; +using Server.Network.Bans; + +namespace Server.Network; + +/// +/// An allowlist addresses earn by logging in successfully, so a reputation feed cannot get a known player +/// blocked and a flaky connection cannot get one globally banned. +/// +/// +/// +/// Consulted only after the blocklist has already matched, and again before a ban is contributed, so a +/// normal accept pays nothing for it. An entry is evidence rather than a licence: enough strikes inside the +/// window revokes it. It cannot bootstrap, so it hedges stable addresses and does not replace +/// . See dev-docs/ip-bans-and-allowlists.md. +/// +/// +/// Both dictionaries are game-loop state. Only the file write runs off-loop, over a snapshot taken on the +/// loop. +/// +/// +public static class LoginAllowlist +{ + private static readonly ILogger logger = LogFactory.GetLogger(typeof(LoginAllowlist)); + + // Address (normalized v6 bits) -> unix seconds of its last successful login. Loop-only. + private static readonly Dictionary _allowed = []; + + // Suppressed contributions in the current window. Only holds allowlisted addresses, so it is bounded by + // _allowed and cannot be grown by an attacker. + private static readonly Dictionary _strikes = []; + + private static bool _enabled; + private static string _path; + private static long _ttlSeconds; + private static int _escalateAfterStrikes; + private static long _strikeWindowSeconds; + private static bool _dirty; + + public static int Count => _allowed.Count; + + private struct Strike + { + public int Count; + public long WindowStart; // unix seconds; 0 means "no window open" + } + + public static void Configure() + { + LoginAllowlistConfiguration.Load(); + var s = LoginAllowlistConfiguration.Settings; + + _enabled = s.Enabled && !string.IsNullOrWhiteSpace(s.File) && s.Ttl > TimeSpan.Zero; + if (!_enabled) + { + return; + } + + _path = Path.IsPathRooted(s.File) ? s.File : Path.Join(Core.BaseDirectory, s.File); + _ttlSeconds = (long)s.Ttl.TotalSeconds; + _escalateAfterStrikes = s.EscalateAfterStrikes; + _strikeWindowSeconds = (long)s.StrikeWindow.TotalSeconds; + } + + public static void Initialize() + { + if (!_enabled) + { + logger.Information("Login allowlist disabled"); + return; + } + + Load(); + + var interval = LoginAllowlistConfiguration.Settings.FlushInterval; + if (interval <= TimeSpan.Zero) + { + interval = TimeSpan.FromMinutes(1); + } + + Timer.DelayCall(interval, interval, Flush); + } + + /// + /// Records a successful authentication. Private addresses are skipped: a LAN or loopback login says + /// nothing about the public internet. + /// + public static void RecordLogin(IPAddress address) => RecordLogin(address, ToUnixSeconds(Core.Now)); + + /// The pure write, split out so policy can be tested without a clock. + internal static void RecordLogin(IPAddress address, long nowUnix) + { + if (!_enabled || address == null || address.IsPrivateNetwork()) + { + return; + } + + var key = address.ToUInt128(); + _allowed[key] = nowUnix; + + // A fresh login clears the tally: someone just proved they hold an account. + _strikes.Remove(key); + _dirty = true; + } + + /// + /// True when this address logged in within the TTL. Expiry is decided on read, so a stale entry left for + /// the next flush can never allow anything. + /// + public static bool IsAllowed(IPAddress address) => IsAllowed(address, ToUnixSeconds(Core.Now)); + + /// The pure decision, split out so the TTL policy can be tested without a clock. + internal static bool IsAllowed(IPAddress address, long nowUnix) + { + if (!_enabled || address == null) + { + return false; + } + + return _allowed.TryGetValue(address.ToUInt128(), out var stamp) && nowUnix - stamp <= _ttlSeconds; + } + + public static bool IsExemptFromEscalation(IPAddress address, string reason) => + IsExemptFromEscalation(address, reason, ToUnixSeconds(Core.Now)); + + /// + /// Whether this contribution should be dropped instead of escalated, counting a strike if so. Not a pure + /// read — calling it is what spends the address's allowance. + /// + internal static bool IsExemptFromEscalation(IPAddress address, string reason, long nowUnix) + { + // An operator's explicit ban, or a reason nobody opted in, escalates untouched. + if (!BanReasons.IsBehavioral(reason) || !IsAllowed(address, nowUnix)) + { + return false; + } + + if (_escalateAfterStrikes <= 0) + { + return true; // revocation disabled: an entry is unconditional + } + + var key = address.ToUInt128(); + _strikes.TryGetValue(key, out var strike); + + if (strike.WindowStart == 0 || nowUnix - strike.WindowStart > _strikeWindowSeconds) + { + strike = new Strike { WindowStart = nowUnix }; + } + + strike.Count++; + + if (strike.Count < _escalateAfterStrikes) + { + _strikes[key] = strike; + return true; + } + + // Allowance spent: drop the entry so this and all after it escalate. Earned back by logging in. + _allowed.Remove(key); + _strikes.Remove(key); + _dirty = true; + + logger.Information( + "{Address} revoked from the login allowlist after {Count} suppressed contribution(s); last was '{Reason}'", + address, + strike.Count, + reason + ); + + return false; + } + + internal static void LoadForTesting(bool enabled, long ttlSeconds, int escalateAfterStrikes = 0, long strikeWindowSeconds = 3600) + { + _allowed.Clear(); + _strikes.Clear(); + _enabled = enabled; + _ttlSeconds = ttlSeconds; + _escalateAfterStrikes = escalateAfterStrikes; + _strikeWindowSeconds = strikeWindowSeconds; + _path = null; + } + + private static long ToUnixSeconds(DateTime utc) => (long)(utc - DateTime.UnixEpoch).TotalSeconds; + + private static void Flush() + { + if (!_enabled || !_dirty) + { + return; + } + + // A save owns the disk and nothing here is urgent. _dirty stays set, so skipping loses nothing. + // See the threading policy in CLAUDE.md (rules #3 and #10). + if (World.Saving || World.WorldState == WorldState.PendingSave) + { + return; + } + + var nowUnix = ToUnixSeconds(Core.Now); + var cutoff = nowUnix - _ttlSeconds; + + // Prune and snapshot in one loop-side pass; the writer only sees private copies. Not pooled: + // STArrayPool is single-threaded and these escape to another thread. + var addresses = new UInt128[_allowed.Count]; + var stamps = new long[_allowed.Count]; + var count = 0; + + using var expired = new PooledRefList(16); + + foreach (var (address, stamp) in _allowed) + { + if (stamp < cutoff) + { + expired.Add(address); + continue; + } + + addresses[count] = address; + stamps[count] = stamp; + count++; + } + + for (var i = 0; i < expired.Count; i++) + { + _allowed.Remove(expired[i]); + _strikes.Remove(expired[i]); + } + + PruneStaleStrikes(nowUnix); + + _dirty = false; + + var path = _path; + var total = count; + var dropped = expired.Count; + + _ = Task.Run(() => Write(path, addresses, stamps, total, dropped)); + } + + /// Drops tallies whose window has closed. + private static void PruneStaleStrikes(long nowUnix) + { + if (_strikes.Count == 0) + { + return; + } + + using var stale = new PooledRefList(16); + + foreach (var (address, strike) in _strikes) + { + if (nowUnix - strike.WindowStart > _strikeWindowSeconds) + { + stale.Add(address); + } + } + + for (var i = 0; i < stale.Count; i++) + { + _strikes.Remove(stale[i]); + } + } + + private static void Write(string path, UInt128[] addresses, long[] stamps, int count, int dropped) + { + try + { + var dir = Path.GetDirectoryName(path); + if (!string.IsNullOrEmpty(dir)) + { + Directory.CreateDirectory(dir); + } + + // Sibling + swap, so a reader never sees a half-written list. + var tmp = path + ".tmp"; + + using (var writer = new StreamWriter(tmp, false, new UTF8Encoding(false), 1 << 16)) + { + writer.Write("# modernuo-login-allowlist generated="); + writer.Write(DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture)); + writer.Write(" count="); + writer.Write(count); + writer.Write('\n'); + + for (var i = 0; i < count; i++) + { + writer.Write(addresses[i].ToIpAddress().ToString()); + writer.Write(' '); + writer.Write(stamps[i]); + writer.Write('\n'); + } + } + + File.Move(tmp, path, true); + + if (dropped > 0) + { + logger.Information("Login allowlist wrote {Count} entr(ies), dropped {Dropped} past TTL", count, dropped); + } + } + catch (Exception e) + { + // Recoverable: entries are still in memory and the next flush retries. + logger.Warning(e, "Could not write the login allowlist to \"{Path}\"", path); + } + } + + private static void Load() + { + if (!File.Exists(_path)) + { + logger.Information("Login allowlist empty: no file at \"{Path}\"", _path); + return; + } + + var cutoff = ToUnixSeconds(Core.Now) - _ttlSeconds; + var loaded = 0; + var skipped = 0; + + try + { + foreach (var line in File.ReadLines(_path)) + { + var span = line.AsSpan().Trim(); + if (span.Length == 0 || span[0] == '#' || span[0] == ';') + { + continue; + } + + var sep = span.IndexOf(' '); + if (sep <= 0 || + !IPAddress.TryParse(span[..sep], out var address) || + !long.TryParse(span[(sep + 1)..].Trim(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var stamp)) + { + skipped++; + continue; + } + + // Expired on disk: do not carry a stranger into memory. + if (stamp < cutoff) + { + skipped++; + _dirty = true; // the file is now out of date; the next flush rewrites it + continue; + } + + _allowed[address.ToUInt128()] = stamp; + loaded++; + } + } + catch (Exception e) + { + // Fail open: an unreadable list allows nobody, which beats refusing to boot. + logger.Warning(e, "Could not read the login allowlist at \"{Path}\"; continuing with {Count}", _path, _allowed.Count); + return; + } + + logger.Information("Login allowlist loaded {Loaded} entr(ies) ({Skipped} expired or malformed)", loaded, skipped); + } +} diff --git a/Projects/UOContent/Network/LoginAllowlist/LoginAllowlistConfiguration.cs b/Projects/UOContent/Network/LoginAllowlist/LoginAllowlistConfiguration.cs new file mode 100644 index 000000000..b3242a85f --- /dev/null +++ b/Projects/UOContent/Network/LoginAllowlist/LoginAllowlistConfiguration.cs @@ -0,0 +1,106 @@ +/************************************************************************* + * ModernUO * + * Copyright 2019-2026 - ModernUO Development Team * + * Email: hi@modernuo.com * + * File: LoginAllowlistConfiguration.cs * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *************************************************************************/ + +using System; +using System.IO; +using System.Text.Json.Serialization; +using Server.Json; + +namespace Server.Network; + +/// +/// Loads the from Configuration/login-allowlist.json (matching +/// the per-feature JSON config pattern used by BlocklistConfiguration). Loaded once; a missing file +/// writes a template so operators have something to edit. +/// +public static class LoginAllowlistConfiguration +{ + private const string _path = "Configuration/login-allowlist.json"; + + public static LoginAllowlistSettings Settings { get; private set; } + + public static void Load() + { + var path = Path.Join(Core.BaseDirectory, _path); + + if (File.Exists(path)) + { + Settings = JsonConfig.Deserialize(path); + } + else + { + Settings = new LoginAllowlistSettings(); + Save(); + } + } + + private static void Save() + { + JsonConfig.Serialize(Path.Join(Core.BaseDirectory, _path), Settings); + } +} + +/// +/// Bound configuration for : which addresses have recently proven they carry a +/// real player, how long that proof counts, and how much misbehaviour revokes it. +/// +/// +/// The recency window is the point. Consumer addresses are reassigned constantly, and on CGNAT the same +/// address fronts a different subscriber week to week, so a list without a TTL becomes a list of strangers. +/// +public record LoginAllowlistSettings +{ + /// Whether successful logins are recorded and consulted at all. Disabled makes the list inert. + [JsonPropertyName("enabled")] + public bool Enabled { get; set; } = true; + + /// + /// Where the list is persisted. A relative path resolves against . + /// Plain text, one address unix-seconds pair per line, so it can be read and edited by hand. + /// Set to "" to disable. + /// + [JsonPropertyName("file")] + public string File { get; set; } = "Configuration/login-allowlist.txt"; + + /// + /// How long a successful login allowlists its address; dropped on the next flush after that. 90 days + /// covers a player who takes a season off without carrying a reassigned address indefinitely. + /// + [JsonPropertyName("ttl")] + public TimeSpan Ttl { get; set; } = TimeSpan.FromDays(90); + + /// + /// How often a changed list is written out. A crash loses at most this much, and an entry is re-earned by + /// the next login. + /// + [JsonPropertyName("flushInterval")] + public TimeSpan FlushInterval { get; set; } = TimeSpan.FromMinutes(1); + + /// + /// How many suppressed contributions inside revoke an address's entry. Past + /// this it escalates like anything else until it earns a new entry by logging in again. + /// + /// + /// Generous on purpose: local defences never stop applying, so a high threshold only delays the external + /// ban. A bad line might trip a gate a few times an hour; a host being used to flood burns through this + /// in seconds. Set to 0 to never revoke. + /// + [JsonPropertyName("escalateAfterStrikes")] + public int EscalateAfterStrikes { get; set; } = 10; + + /// Rolling window the strike count is measured over. A quiet hour clears the tally. + [JsonPropertyName("strikeWindow")] + public TimeSpan StrikeWindow { get; set; } = TimeSpan.FromHours(1); +} diff --git a/dev-docs/ip-bans-and-allowlists.md b/dev-docs/ip-bans-and-allowlists.md new file mode 100644 index 000000000..5588bb19e --- /dev/null +++ b/dev-docs/ip-bans-and-allowlists.md @@ -0,0 +1,216 @@ +# IP Bans, Blocklists and Allowlists + +How a shard decides to refuse a connection, how it contributes bans to an external bouncer, and how an +operator exempts someone who was caught by mistake. + +If you are here because **a player cannot connect**, skip to [Unblocking a player](#unblocking-a-player). + +## The shape of it + +Two independent questions, deliberately separated: + +| Question | Answered by | Effect | +|---|---|---| +| Refuse this connection? | `IConnectionFilter` implementations, at accept | The socket is dropped | +| Tell the outside world about it? | `BanChannel` → `IBanReporter` implementations | CrowdSec, and from there an OS bouncer | + +`BanChannel` **never enforces** and filters **never report on each other's behalf**. Enforcement that +outlives the process belongs to the OS bouncer; the shard only contributes. + +### Refusing a connection + +Filters are consulted in registration order, first denial wins: + +| Filter | Source | Scope | +|---|---|---| +| `firewall` | `Configuration/firewall.json`, mutable in-game | Admin-curated, permanent | +| `blocklist` | `Configuration/ip-blocklist.txt` (millions of entries) | Reputation feeds | +| `auto-denylist` | In-memory, 15 min | What this shard just caught misbehaving | + +### Contributing a ban + +`BanChannel.Report(address, ttl, reason)` → `BanExemptions.IsExempt` → if not exempt, fan out to every +reporter (`crowdsec`, `auto-denylist`). + +### Allowlists + +Two, with different authority: + +| List | Source | Revocable? | Covers | +|---|---|---|---| +| `FileAllowlist` | every `ip-allowlist*.txt` | No — an operator said so | Blocking **and** escalation | +| `LoginAllowlist` | Earned by authenticating, 90-day TTL | Yes — 10 strikes/hour | Blocking **and** escalation | + +Both are consulted **only after the blocklist has already matched**, so a normal accept — the one an +attacker is trying to flood — pays nothing for them. The accept gate itself is deliberately allowlist-free: +a whitelist there could only turn a deny into an allow at the cost of a lookup on every accept, the +attacker's included. + +## Unblocking a player + +### 1. Find out what is actually blocking them + +```bash +# In the reputation blocklist? +grep -x "203.0.113.42" Distribution/Configuration/ip-blocklist.txt + +# A live external decision? (this is what survives a restart) +cscli decisions list --ip 203.0.113.42 + +# Admin-curated? +grep 203.0.113.42 Distribution/Configuration/firewall.json +``` + +If none of those match, they may be inside a **CIDR** in the blocklist, or held by the in-memory +`auto-denylist` — that one is not queryable and expires on its own within 15 minutes. + +### 2. Add them to the allowlist + +One entry per line in `Distribution/Configuration/ip-allowlist.txt` — a bare address or a CIDR. This file +is yours; the generator creates it once and never rewrites it. + +``` +203.0.113.42 # shard owner, listed via a shared upstream address +198.51.100.0/24 # a whole range if the ISP rotates within it +``` + +The shard reloads within `reloadInterval` (60s default). **No restart, and no need to re-run the +generator.** From that point the address is neither blocked nor contributed. + +### 3. Clear any ban that already exists + +A config change cannot retract a ban that has already left the building: + +```bash +cscli decisions delete -i 203.0.113.42 +``` + +### 4. If it keeps coming back + +The shard is no longer contributing them, so a recurring ban is coming from CrowdSec's own sources (the +community blocklist, another watcher). Allowlist it there too: + +```bash +cscli allowlists create shard-staff -d "Known-good player addresses" +cscli allowlists add shard-staff 203.0.113.42 +``` + +### What will NOT work + +- **Deleting the CrowdSec decision alone.** If the address is still in `ip-blocklist.txt` and not + allowlisted, the next connection re-reports it within `promoteSuppression` (60s). +- **Editing `ip-blocklist.txt` by hand.** The next generator run rewrites the whole file. +- **`cscli allowlists` alone.** That is the enforcement layer. The shard's own accept gate sits upstream of + it and will still refuse the connection. + +## Why entries appear that should not + +Reputation feeds list shared consumer address space constantly. On CGNAT one public address fronts many +subscribers **at the same time**, so a single abusive customer gets the address listed and everyone else +behind it is blocked with them — and where leases rotate, a listing says little about whoever holds the +address now. This is near-universal on mobile carriers, satellite (Starlink) and WISPs, and common on +fixed-line broadband outside North America. + +A **carve-out** exempts a whole network. **None ship with ModernUO** — which providers to exempt depends on +where your players actually are, and a carve-out names a real network, so you build the ones you need: + +```powershell +# Exempt a CGNAT provider whose players keep getting listed +.\Export-IpBlocklist.ps1 -AddCarveout starlink -Asn 14593 + +# Later: bring every carve-out up to date with what those networks currently announce +.\Export-IpBlocklist.ps1 -RefreshCarveouts +``` + +That writes `ip-allowlist-starlink.txt` beside the blocklist, and every `ip-allowlist*.txt` there is +subtracted — both by the generator and by the shard, with no config edit. Starlink costs about 0.1% of the +list. Blank a file (keep the file) to reputation-block that network again; delete it to drop the carve-out. + +Carve-out files carry an `asn=` marker in their header, which is how `-RefreshCarveouts` finds them. A +hand-written allowlist has no marker and is never rewritten. + +**Do you need one?** If players report being blocked and they are on satellite, mobile, or an ISP short on +IPv4, probably yes. Find the ASN by looking up an address the network hands out on any public BGP lookup. + +Prefixes come from **announcements, not ownership records**. Registry data disagrees with what is actually +routed and silently caps result sets: ARIN whois returns at most 256 rows and gives per-customer /24s, and +`206.83.96.0/19` reads as APNIC in RDAP even though `206.83.96/21` is announced by Starlink. + +## Behavioural detection + +Verdicts the shard reaches by watching a connection, rather than by consulting a list: + +| Reason | Trigger | +|---|---| +| `rate-limit` | Too many connection attempts in the limiter's window | +| `silent-connect` | Reaped after `ConnectingSocketIdleLimit` (5s) having sent **zero bytes** | +| `invalid-seed` | Opened with a zero seed, which no real client sends | +| `foreign-protocol` | Positively identified as HTTP, TLS or SSH | + +`BanReasons.IsBehavioral` gates two things: only these may be exempted, and only these enter the +`auto-denylist`. It is an **opt-in list**, not "everything except `manual`" — a reason added later escalates +normally rather than silently inheriting an exemption. `manual` is never exempt and never auto-denied. + +Escalation is **immediate**, on the first detection: a 15-minute local hold plus a `badConnectDuration` +(4h) contribution. There is no N-connection threshold; the strike counter governs only revoking a +`LoginAllowlist` entry. + +### What is deliberately NOT detected + +**Do not add rules based on arrival framing.** TCP has no message boundaries, so the network, the OS or a +middlebox can split the opening bytes anywhere regardless of what the client sent. A rule of the form +"these bytes must arrive together" is broken by construction and drops real players on poor links. This has +been tried and reverted before. + +**Do not treat unreadable payloads as hostile.** A legitimate client with encryption enabled when the shard +expects none sends a structurally perfect connection whose payload is noise — `LoginEncryption.ClientDecrypt` +is a byte-for-byte stream XOR, so it preserves length exactly while destroying content. This is why +detection asks "is this positively some *other* protocol?" rather than "is this a good UO client?": however +misconfigured a UO client is, it never sends `GET / HTTP/1.1`. + +**Timeouts are keyed on bytes-received, not elapsed time.** A connection that sent *something* and ran out +of time is far more likely a slow link than an attack. Banning those produces the worst failure mode +available: the player retries, trips the rate limiter, and compounds a bad connection into hours of being +firewalled off. Shortening the 5s handshake window has been tried and broke real players. + +## Known limits + +- **An allowlist cannot bootstrap.** A `LoginAllowlist` entry is only earned by getting in, so it can never + repair an existing false positive, and it is weakest on rotating CGNAT — a player whose lease moved is a + stranger again. `FileAllowlist` is the fix for that, which is why it is manual. +- **A never-logged-in player on a shared address can still be caught**, for up to `badConnectDuration`, if a + co-tenant misbehaves. Accepted: it is 4h and self-healing. The cheapest lever is `badConnectDuration`. +- **`MaxConnections` (4096) is a hard ceiling.** The accept gate runs *after* the kernel completed the TCP + handshake, so a blocklist match saves the socket setup and the `NetState` slot but never the connection + itself. Only an upstream L4 proxy or edge scrubbing moves that cost off the shard. + +## Configuration + +| File | Controls | +|---|---| +| `bans.json` | `reportRateLimitTrips`, `autoBanDuration`, `reportBadConnects`, `badConnectDuration` | +| `blocklist.json` | `file`, `allowlistFiles` (wildcards allowed), `reloadInterval`, `reportHits`, `banDuration`, `promoteSuppression` | +| `login-allowlist.json` | `enabled`, `file`, `ttl`, `flushInterval`, `escalateAfterStrikes`, `strikeWindow` | +| `auto-denylist.json` | `enabled`, `duration`, `maxEntries` | +| `crowdsec.json` | `lapiUrl`, `machineId`, `password`, `origin`, `manualBanDuration`, `flushInterval`, `maxQueue` | +| `firewall.json` | Admin-curated entries | + +A shard fronted by an upstream proxy can disable all of it and register nothing. + +## Key files + +| File | Role | +|---|---| +| `Projects/Server/Network/IConnectionFilter.cs` | Accept-path gate contract | +| `Projects/Server/Network/ConnectionFilters.cs` | Filter registry + lifecycle | +| `Projects/Server/Network/ForeignProtocol.cs` | Positive identification of non-UO traffic | +| `Projects/Server/Network/Bans/BanChannel.cs` | Contribution fan-out + `IsExempt` seam | +| `Projects/Server/Network/Bans/BanReasons.cs` | Reason slugs + the behavioural opt-in set | +| `Projects/UOContent/Network/BanExemptions.cs` | Combines both allowlists into one answer | +| `Projects/UOContent/Network/Blocklist/BlocklistFilter.cs` | File-sourced blocklist filter | +| `Projects/UOContent/Network/Blocklist/FileAllowlist.cs` | Operator carve-outs, read from the allowlist files | +| `Projects/UOContent/Network/LoginAllowlist/LoginAllowlist.cs` | Allowlist earned by authenticating | +| `Projects/UOContent/Network/AutoDenylist/AutoDenylist.cs` | Short-lived local hold | +| `Projects/UOContent/Network/CrowdSec/CrowdSecReporter.cs` | LAPI contribution sink | +| `Projects/UOContent/Network/Firewall/Firewall.cs` | Admin-curated firewall set | +| `tools/Export-IpBlocklist.ps1` | Blocklist generator + allowlist subtraction | diff --git a/dev-docs/networking-packets.md b/dev-docs/networking-packets.md index 5eed62249..af47a1e06 100644 --- a/dev-docs/networking-packets.md +++ b/dev-docs/networking-packets.md @@ -509,10 +509,15 @@ Rules: - A filter that throws is **unregistered** and the connection fails open. A filter that faults once faults for every connection, so leaving it registered would mean an exception per accept. -Core owns the question; **every implementation lives in UOContent**. The two that ship are `firewall` -(admin-curated, mutable at runtime, persisted to `Configuration/firewall.json`) and `blocklist` -(file-sourced, millions of entries, demand-pages hits to CrowdSec). A shard that fronts its server with -an upstream proxy or edge scrubbing can drop both and register nothing. +Core owns the question; **every implementation lives in UOContent**. The three that ship are `firewall` +(admin-curated, mutable at runtime, persisted to `Configuration/firewall.json`), `blocklist` (file-sourced, +millions of entries, demand-pages hits to CrowdSec) and `auto-denylist` (in-memory, short-lived, fed by the +shard's own behavioural detections). A shard that fronts its server with an upstream proxy or edge scrubbing +can drop all of them and register nothing. + +The allowlists, ban contribution, behavioural detection and the operator process for exempting a +false-positive address are covered separately in +[`dev-docs/ip-bans-and-allowlists.md`](ip-bans-and-allowlists.md). Do **not** route this kind of check through `EventSink.InvokeSocketConnect` -- that fires later and allocates a `SocketConnectEventArgs` per connection, which is exactly what the accept path avoids for @@ -553,6 +558,10 @@ those, so the helpers check both. | `Projects/Server/Network/PacketHandler.cs` | PacketHandler class | | `Projects/Server/Network/IConnectionFilter.cs` | Accept-path gate contract | | `Projects/Server/Network/ConnectionFilters.cs` | Filter registry + lifecycle | -| `Projects/UOContent/Misc/Firewall/Firewall.cs` | Admin-curated firewall set | +| `Projects/UOContent/Network/Firewall/Firewall.cs` | Admin-curated firewall set | | `Projects/Server/Utilities/IPAddressUtility.cs` | IPAddress <-> UInt128 normalization, CIDR parsing | -| `Projects/UOContent/Misc/Blocklist/BlocklistFilter.cs` | File-sourced blocklist filter | +| `Projects/UOContent/Network/Blocklist/BlocklistFilter.cs` | File-sourced blocklist filter | +| `Projects/UOContent/Network/LoginAllowlist/LoginAllowlist.cs` | Allowlist earned by a recent successful login | +| `Projects/UOContent/Network/AutoDenylist/AutoDenylist.cs` | Short-lived local hold on behavioural detections | +| `Projects/Server/Network/Bans/BanReasons.cs` | Ban reason slugs + the behavioural opt-in set | +| `Projects/Server/Network/ForeignProtocol.cs` | Positive identification of non-UO traffic (HTTP/TLS/SSH) | diff --git a/tools/Export-IpBlocklist.ps1 b/tools/Export-IpBlocklist.ps1 index 81b9afd06..0e9fd781d 100644 --- a/tools/Export-IpBlocklist.ps1 +++ b/tools/Export-IpBlocklist.ps1 @@ -34,7 +34,35 @@ player -- and those are barely present here anyway (bitwire is ~5% of VPN-tunnel lists). If you ever want to protect VPN/Tor players, pass -ExcludeAnonymizers to subtract Tor/open-proxy/VPN IPs from the output. - OUTPUT FORMAT (must stay in sync with UOContent/Misc/Blocklist/BlocklistFile.cs): + ALLOWLIST + Aggregators inevitably list shared consumer address space. A CGNAT public IP fronts many subscribers, so + one abusive customer gets the address listed and every other subscriber behind it is blocked with them. + Entries in the allowlist file (-AllowlistFile) are SUBTRACTED from the merged set before it is written, so + the exemption costs nothing on the shard's accept path -- which is deliberately allowlist-free, because a + whitelist there could only ever turn a deny into an allow at the price of a lookup on every accept, the + attacker's included. Allowlisting belongs here (at generation) and at the enforcement layer + (`cscli allowlists`), never at the gate. + + Subtraction is range-correct: an allowlisted address that falls inside a blocked CIDR splits that CIDR + around the hole instead of being silently ignored, so an exemption always takes effect no matter which + shape the feed happened to publish. + + Every `ip-allowlist*.txt` beside the output is subtracted, so allowlists are split by owner rather than + kept in one file: `ip-allowlist.txt` holds the operator's own exemptions and is never rewritten, while + network carve-outs live in `ip-allowlist-.txt`. Keeping them apart means a carve-out can be + regenerated, diffed or copied to another shard without disturbing hand-written entries. + + NO CARVE-OUT IS SHIPPED. Which providers to exempt is a policy call that depends on where a shard's + players actually are, and a carve-out names a real network, so this script builds them on request rather + than publishing anyone's. A shard whose players are on CGNAT -- satellite, mobile, or an ISP short on + IPv4 -- will usually want one: + + .\Export-IpBlocklist.ps1 -AddCarveout starlink -Asn 14593 + + That costs roughly 0.1% of the list. Abusive hosts inside a carved-out network are still caught on + BEHAVIOR by the rate limiter and promoted to CrowdSec, which is the gate that actually observes them. + + OUTPUT FORMAT (must stay in sync with UOContent/Network/Blocklist/BlocklistFile.cs): Line 1 is a header comment carrying the version markers, e.g. # modernuo-blocklist generated=2026-07-25T18:03:11Z count=3914022 ipv4=3901188 cidr=12834 The shard polls `reloadInterval` and reloads when the file mtime AND `generated=` change, @@ -84,6 +112,46 @@ .PARAMETER Feeds Which feeds to include (by Name). Default: all of them. +.PARAMETER AllowlistFile + One or more lists of addresses that must NEVER be blocked; every entry is subtracted from the merged set + before the output is written. Same format as the blocklist: one bare IPv4 or CIDR per line, `#`/`;` + comments ignored. + + Defaults to every `ip-allowlist*.txt` beside the output, merged into one allow set: + ip-allowlist.txt operator exemptions -- created on first run, never rewritten + ip-allowlist-.txt a network carve-out -- generated data, safe to regenerate or copy + Discovered rather than configured, so a carve-out you add is picked up with no further edits. Blank a + file (keep the file) to disable its contents; delete it to drop it entirely. + + Passing this parameter replaces the defaults entirely; an explicitly-named file that does not exist is a + warning rather than a silent template write, so a typo cannot look like it worked. Pass `''` to disable + subtraction altogether. + + Editing any allowlist also bypasses -MinInterval on the next run: an exemption you just added would + otherwise sit unapplied for up to the cooldown, which reads exactly like the allowlist not working. + +.PARAMETER AddCarveout + Build a carve-out for a network and start subtracting it. Takes a short name for the file and -Asn for + the network, fetches that ASN's current routing announcements, collapses them, and writes + `ip-allowlist-.txt` beside the output. Implies -Force. + + .\Export-IpBlocklist.ps1 -AddCarveout starlink -Asn 14593 + + No carve-out ships with this script: naming a network to exempt is a policy call for the shard, so they + are built on request rather than published here. Re-running with the same name rebuilds the file. + +.PARAMETER Asn + Autonomous system number for -AddCarveout, e.g. 14593 for Starlink. Look one up by querying an address + the network hands out, or on any public BGP lookup. + +.PARAMETER RefreshCarveouts + Re-fetch every carve-out beside the output and rewrite it from current routing announcements. Implies + -Force. Carve-outs are recognised by the `asn=` marker in their header, so a hand-written allowlist is + left alone, and one whose fetch fails keeps the data it already had. + + Announcements rather than ownership records on purpose: registry data disagrees with what is actually + routed, and registry queries silently cap their result sets. + .PARAMETER ExcludeAnonymizers Also download Tor-exit / open-proxy / VPN-tunnel lists and SUBTRACT those IPs from the output. Off by default -- for a game server, Tor/open-proxy relays are attack infrastructure you want to block. Turn @@ -106,6 +174,21 @@ # Regenerate right now, ignoring the cooldown. .\Export-IpBlocklist.ps1 -DistributionPath 'C:\Shard\Distribution' -Force +.EXAMPLE + # Unblock a player caught by a shared-IP listing: add the address, then regenerate. Editing the + # allowlist bypasses the cooldown, so no -Force is needed. + Add-Content 'C:\Shard\Distribution\Configuration\ip-allowlist.txt' '203.0.113.42' + .\Export-IpBlocklist.ps1 -DistributionPath 'C:\Shard\Distribution' + +.EXAMPLE + # Check what an allowlist would cost before committing to it. + .\Export-IpBlocklist.ps1 -AllowlistFile 'D:\shared\allow.txt' -DryRun + +.EXAMPLE + # Exempt a CGNAT provider whose players keep getting listed, then keep it current. + .\Export-IpBlocklist.ps1 -AddCarveout starlink -Asn 14593 + .\Export-IpBlocklist.ps1 -RefreshCarveouts + .EXAMPLE # Linux/macOS, e.g. from cron: pwsh -File /opt/modernuo/Export-IpBlocklist.ps1 -DistributionPath /opt/modernuo/Distribution @@ -120,6 +203,10 @@ param( [string] $DistributionPath, [string] $OutFile, [string] $MinInterval = '2h', + [string[]] $AllowlistFile, + [string] $AddCarveout, + [int] $Asn, + [switch] $RefreshCarveouts, [string[]] $Feeds, [switch] $ExcludeAnonymizers, [switch] $Force, @@ -127,6 +214,11 @@ param( ) $ErrorActionPreference = 'Stop' + +# Fail before any download rather than after 60MB of feeds. +if ($AddCarveout -and $Asn -le 0) { + throw "-AddCarveout needs -Asn, e.g. -AddCarveout starlink -Asn 14593" +} $UA = 'ModernUO-Blocklist-Export' $totalSw = [System.Diagnostics.Stopwatch]::StartNew() @@ -153,6 +245,132 @@ if (-not $OutFile) { $OutFile = Join-Path $DistributionPath @DefaultPathSegments } +# --------------------------------------------------------------------------------------------------------- +# Network carve-outs are DATA, not code: this script ships none. Which providers a shard exempts is a policy +# call that depends on where its players actually are, so the carve-outs live in files an admin creates with +# -AddCarveout, and every ip-allowlist*.txt beside the output is subtracted. +# +# A shard whose players are on CGNAT -- satellite, mobile, or an ISP short on IPv4 -- will usually want one: +# .\Export-IpBlocklist.ps1 -AddCarveout starlink -Asn 14593 +# --------------------------------------------------------------------------------------------------------- + +# --------------------------------------------------------------------------------------------------------- +# Resolve the allowlists. They sit beside the output by default so relocating the blocklist keeps the set +# together, and they are split by owner: `ip-allowlist.txt` is the operator's -- hand-edited, never rewritten +# -- while each carve-out file is generated data that can be regenerated, diffed or copied between shards +# without touching anyone's local exemptions. +# +# Carve-outs are discovered rather than listed, so a file an admin drops in is picked up with no config edit +# and no code change. An EXPLICIT -AllowlistFile replaces the whole set and is never templated: if the +# operator names a file, a missing one is a typo worth hearing about. +# --------------------------------------------------------------------------------------------------------- +$AllowGlob = 'ip-allowlist*.txt' +$ConfigDir = Split-Path -Parent $OutFile + +$allowExplicit = $PSBoundParameters.ContainsKey('AllowlistFile') + +if ($allowExplicit) { + $AllowPaths = @($AllowlistFile | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }) +} +else { + $AllowPaths = @(Get-ChildItem -Path $ConfigDir -Filter $AllowGlob -File -ErrorAction SilentlyContinue | + Sort-Object Name | ForEach-Object { $_.FullName }) +} + +function Write-AllowlistFile { + param([string]$Path, [string[]]$Lines) + + $dir = Split-Path -Parent $Path + if ($dir -and -not (Test-Path -LiteralPath $dir -PathType Container)) { + New-Item -ItemType Directory -Path $dir -Force | Out-Null + } + + # Same atomic write the blocklist gets: a half-written allowlist would silently under-subtract. + $tmp = $Path + '.tmp' + [IO.File]::WriteAllLines($tmp, $Lines, [System.Text.UTF8Encoding]::new($false)) + [IO.File]::Move($tmp, $Path, $true) +} + +$OperatorTemplate = @' +# ModernUO blocklist allowlist -- every entry here is SUBTRACTED from the generated blocklist. +# +# This file is yours. Export-IpBlocklist.ps1 creates it once and never rewrites it, so anything you add +# survives every regeneration. +# +# One entry per line: a bare IPv4 address (1.2.3.4) or a CIDR (1.2.3.0/24). Lines starting with '#' or ';' +# are comments. Order does not matter. Re-run Export-IpBlocklist.ps1 to apply changes -- editing this file +# bypasses the -MinInterval cooldown, so no -Force is needed. +# +# Removal is range-correct: an address listed here is removed even when a feed published it as part of a +# larger CIDR -- that CIDR is split around the hole rather than dropped wholesale or silently ignored. +# +# Network carve-outs live in their own ip-allowlist-.txt beside this one (see -AddCarveout), so they +# can be regenerated or copied between shards without touching anything you put here. +# +# Put player/staff exemptions below, one per line, e.g.: +# 203.0.113.42 # shard owner, listed via a shared upstream address +'@ + +# Carve-out files carry their own `asn=` marker, so -RefreshCarveouts can rebuild whatever an admin created +# without this script keeping a list of anyone's networks. +function Get-CarveoutHeader { + param([string]$Name, [int]$CarveoutAsn) + + @( + ("# {0} carve-out (asn={1}) -- subtracted from the generated blocklist." -f $Name, $CarveoutAsn) + "#" + "# Reputation feeds list shared consumer address space constantly, so a hit inside a CGNAT network" + "# says little about the player currently behind it. Abusive hosts here are still caught on BEHAVIOR." + "#" + "# GENERATED DATA -- safe to regenerate, diff, or copy to another shard. Blank the file (keep the" + "# file) to reputation-block this network again; delete it to stop carving it out entirely." + "#" + "# Refresh with: .\Export-IpBlocklist.ps1 -RefreshCarveouts" + ) +} + +# Fetches a network's currently ANNOUNCED prefixes and collapses them. Routing data, not a registry: +# ownership records disagree with what is actually announced, and registry queries cap their result sets. +function Get-CarveoutPrefixes { + param([int]$CarveoutAsn) + + $url = "https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS$CarveoutAsn" + $json = Get-Url -Url $url -Label ("AS{0} prefixes" -f $CarveoutAsn) | ConvertFrom-Json + + $v4 = @($json.data.prefixes.prefix | Where-Object { $_ -and $_ -notmatch ':' }) + if (-not $v4) { throw "AS$CarveoutAsn announced no IPv4 prefixes -- refusing to overwrite the carve-out." } + + [BlocklistExporter]::CollapsePrefixes(($v4 -join "`n")) +} + +# Reads the `asn=` marker back out of a carve-out file. Anything without one is a hand-written allowlist and +# is left alone by -RefreshCarveouts. +function Get-CarveoutAsn { + param([string]$Path) + + foreach ($line in (Get-Content -LiteralPath $Path -TotalCount 5 -ErrorAction SilentlyContinue)) { + if ($line -match 'asn=(\d+)') { return [int]$Matches[1] } + } + + return 0 +} + +# The operator's own list is the one file this script will create unprompted; carve-outs are opt-in. +if (-not $allowExplicit) { + $operatorPath = Join-Path $ConfigDir 'ip-allowlist.txt' + if (-not (Test-Path -LiteralPath $operatorPath -PathType Leaf)) { + Write-AllowlistFile -Path $operatorPath -Lines @($OperatorTemplate) + Write-Host ("Created allowlist at {0} (add player/staff exemptions here)." -f $operatorPath) + $AllowPaths = @($operatorPath) + $AllowPaths + } +} +else { + $AllowPaths = @($AllowPaths | ForEach-Object { + if (Test-Path -LiteralPath $_ -PathType Leaf) { return $_ } + Write-Warning ("Allowlist '{0}' does not exist -- nothing will be subtracted from it. Check the path." -f $_) + }) +} + # --------------------------------------------------------------------------------------------------------- # Cooldown gate. Runs BEFORE anything is downloaded: the whole point is that a misconfigured scheduler or a # retry loop cannot spam the upstream feeds. State lives in the output file itself (`generated=` header, @@ -215,17 +433,40 @@ function Get-BlocklistAge { } $minAge = ConvertTo-Duration $MinInterval -if (-not $Force -and $minAge -gt [TimeSpan]::Zero) { +# Asking for carve-out data implies -Force: waiting out the cooldown and leaving the old data in place would +# be the wrong answer. +if (-not $Force -and -not $RefreshCarveouts -and -not $AddCarveout -and $minAge -gt [TimeSpan]::Zero) { $existing = Get-BlocklistAge -Path $OutFile if ($existing) { # A negative age means the stamp is in the future (clock skew, or a file from another host). Treat it # as fresh: refusing to run is the recoverable failure, hammering the feeds on every tick is not. if ($existing.Age -lt $minAge) { - $agoText = if ($existing.Age -lt [TimeSpan]::Zero) { 'in the future -- check the clock' } else { ("{0:N1}h ago" -f $existing.Age.TotalHours) } - Write-Host ("Blocklist at {0} was generated {1} ({2}={3}); newer than -MinInterval {4}." -f ` - $OutFile, $agoText, $existing.Source, $existing.Stamp, $MinInterval) - Write-Host "Nothing downloaded. Pass -Force to regenerate now, or lower -MinInterval." - return + # An allowlist edited since the list was built is the one case where waiting out the cooldown is + # the wrong answer: the operator is unblocking someone, and "nothing happened" is indistinguishable + # from the allowlist not working. Cheap to honour -- it can only ever shrink the output. + $changedAllow = $null + $builtAt = [DateTime]::UtcNow - $existing.Age + foreach ($p in $AllowPaths) { + try { + if ((Get-Item -LiteralPath $p -ErrorAction Stop).LastWriteTimeUtc -gt $builtAt) { + $changedAllow = $p + break + } + } + catch { } + } + + if ($changedAllow) { + Write-Host ("Allowlist {0} changed since the blocklist was built; regenerating despite -MinInterval {1}." -f ` + $changedAllow, $MinInterval) + } + else { + $agoText = if ($existing.Age -lt [TimeSpan]::Zero) { 'in the future -- check the clock' } else { ("{0:N1}h ago" -f $existing.Age.TotalHours) } + Write-Host ("Blocklist at {0} was generated {1} ({2}={3}); newer than -MinInterval {4}." -f ` + $OutFile, $agoText, $existing.Source, $existing.Stamp, $MinInterval) + Write-Host "Nothing downloaded. Pass -Force to regenerate now, or lower -MinInterval." + return + } } } } @@ -328,6 +569,244 @@ public static class BlocklistExporter return added; } + // --------------------------------------------------------------------------------------------------- + // Allowlist subtraction. Allow entries become sorted, merged [start,end] ranges once; the blocklist is + // then filtered against them. The interesting case is a blocked CIDR that only PARTIALLY overlaps an + // allow range -- dropping it whole would unblock far more than asked, keeping it whole would ignore the + // exemption, so it is split into the surviving pieces and re-emitted as minimal CIDRs. + // --------------------------------------------------------------------------------------------------- + public sealed class RangeSet + { + public uint[] Start; + public uint[] End; + public int Count; + } + + public sealed class AllowResult + { + public int SinglesRemoved; + public int CidrsDropped; + public int CidrsSplit; + public int EntriesAdded; + } + + // Parses "a.b.c.d/p" to an inclusive range. The base is masked to the prefix, so a sloppy 1.2.3.5/24 + // means the whole 1.2.3.0/24 -- the standard reading, and the safe direction for an exemption. + static bool TryCidrRange(string c, out ulong lo, out ulong hi) + { + lo = 0; hi = 0; + int slash = c.IndexOf('/'); + if (slash <= 0) return false; + uint ip; + if (!TryParseIPv4(c, 0, slash, out ip)) return false; + int bits = 0, bd = 0; + for (int i = slash + 1; i < c.Length; i++) + { + char ch = c[i]; + if (ch < '0' || ch > '9') { bd = -1; break; } + bits = bits * 10 + (ch - '0'); bd++; + } + if (bd <= 0 || bits > 32) return false; + // 1u << 32 is undefined in C# (the shift count is masked to 5 bits), so /0 is special-cased. + uint mask = (bits == 0) ? 0u : ~((uint)((1UL << (32 - bits)) - 1UL)); + ulong size = (bits == 0) ? 0x100000000UL : (1UL << (32 - bits)); + lo = ip & mask; + hi = lo + size - 1UL; + return true; + } + + public static RangeSet BuildRanges(HashSet singles, HashSet cidrs) + { + uint[] s = new uint[singles.Count + cidrs.Count]; + uint[] e = new uint[s.Length]; + int k = 0; + + foreach (uint v in singles) { s[k] = v; e[k] = v; k++; } + foreach (string c in cidrs) + { + ulong lo, hi; + if (!TryCidrRange(c, out lo, out hi)) continue; + s[k] = (uint)lo; + e[k] = (uint)(hi > 0xFFFFFFFFUL ? 0xFFFFFFFFUL : hi); + k++; + } + + Array.Resize(ref s, k); + Array.Resize(ref e, k); + Array.Sort(s, e); + + // Coalesce overlapping AND adjacent ranges so the lookups below can assume disjoint, ordered spans. + int w = 0; + for (int i = 0; i < k; i++) + { + if (w > 0 && (ulong)s[i] <= (ulong)e[w - 1] + 1UL) + { + if (e[i] > e[w - 1]) e[w - 1] = e[i]; + } + else + { + s[w] = s[i]; e[w] = e[i]; w++; + } + } + + return new RangeSet { Start = s, End = e, Count = w }; + } + + // Index of the first range whose End >= v (ranges are disjoint and sorted, so End is sorted too). + static int FirstEndAtLeast(uint[] re, int n, uint v) + { + int lo = 0, hi = n; + while (lo < hi) + { + int mid = (int)(((uint)lo + (uint)hi) >> 1); + if (re[mid] < v) lo = mid + 1; else hi = mid; + } + return lo; + } + + static bool Covered(uint[] rs, uint[] re, int n, uint v) + { + int i = FirstEndAtLeast(re, n, v); + return i < n && rs[i] <= v; + } + + static string FormatCidr(uint ip, int bits) + { + char[] buf = new char[19]; + int p = 0; + p = WriteOctet(buf, p, (ip >> 24) & 255); buf[p++] = '.'; + p = WriteOctet(buf, p, (ip >> 16) & 255); buf[p++] = '.'; + p = WriteOctet(buf, p, (ip >> 8) & 255); buf[p++] = '.'; + p = WriteOctet(buf, p, ip & 255); buf[p++] = '/'; + p = WriteOctet(buf, p, (uint)bits); + return new string(buf, 0, p); + } + + // Writes [lo,hi] as the minimal set of aligned CIDR blocks. A /32 goes back to the singles set so the + // output keeps the file's convention of bare addresses for single hosts. + static void Emit(ulong lo, ulong hi, HashSet singles, HashSet cidrs, AllowResult r) + { + while (lo <= hi) + { + int bits = 32; + while (bits > 0) + { + ulong size = 1UL << (32 - (bits - 1)); + if ((lo % size) != 0UL) break; + if (lo + size - 1UL > hi) break; + bits--; + } + + if (bits == 32) + { + if (singles.Add((uint)lo)) r.EntriesAdded++; + } + else if (cidrs.Add(FormatCidr((uint)lo, bits))) + { + r.EntriesAdded++; + } + + lo += 1UL << (32 - bits); + } + } + + public static AllowResult ApplyAllowlist(HashSet singles, HashSet cidrs, RangeSet allow) + { + var r = new AllowResult(); + if (allow == null || allow.Count == 0) return r; + + uint[] rs = allow.Start, re = allow.End; + int n = allow.Count; + + // Singles first: the CIDR pass below can add new singles, and those are outside the allow ranges by + // construction, so re-testing them would be wasted work. + uint[] sarr = new uint[singles.Count]; + singles.CopyTo(sarr); + for (int i = 0; i < sarr.Length; i++) + { + if (Covered(rs, re, n, sarr[i]) && singles.Remove(sarr[i])) r.SinglesRemoved++; + } + + string[] carr = new string[cidrs.Count]; + cidrs.CopyTo(carr); + cidrs.Clear(); + + for (int i = 0; i < carr.Length; i++) + { + string c = carr[i]; + ulong lo, hi; + + // Unparseable entries are kept verbatim rather than dropped: this pass exists to subtract, and + // silently discarding something it could not read would weaken the list. + if (!TryCidrRange(c, out lo, out hi)) { cidrs.Add(c); continue; } + if (hi > 0xFFFFFFFFUL) hi = 0xFFFFFFFFUL; + + int idx = FirstEndAtLeast(re, n, (uint)lo); + if (idx >= n || (ulong)rs[idx] > hi) { cidrs.Add(c); continue; } // no overlap: the common case + + ulong cursor = lo; + int before = r.EntriesAdded; + for (int j = idx; j < n && (ulong)rs[j] <= hi; j++) + { + if ((ulong)rs[j] > cursor) Emit(cursor, (ulong)rs[j] - 1UL, singles, cidrs, r); + ulong next = (ulong)re[j] + 1UL; + if (next > cursor) cursor = next; + if (cursor > hi) break; + } + if (cursor <= hi) Emit(cursor, hi, singles, cidrs, r); + + if (r.EntriesAdded == before) r.CidrsDropped++; else r.CidrsSplit++; + } + + return r; + } + + static string FormatIp(uint ip) + { + char[] buf = new char[16]; + int p = 0; + p = WriteOctet(buf, p, (ip >> 24) & 255); buf[p++] = '.'; + p = WriteOctet(buf, p, (ip >> 16) & 255); buf[p++] = '.'; + p = WriteOctet(buf, p, (ip >> 8) & 255); buf[p++] = '.'; + p = WriteOctet(buf, p, ip & 255); + return new string(buf, 0, p); + } + + // Collapses a prefix list into the minimal equivalent set, in ascending order. Used by + // -RefreshCarveouts: routing data publishes thousands of overlapping announcements. + public static string[] CollapsePrefixes(string content) + { + uint[] noBogon = new uint[0]; + var singles = new HashSet(); + var cidrs = new HashSet(); + AddContent(content, singles, cidrs, noBogon, noBogon); + + var ranges = BuildRanges(singles, cidrs); + var result = new List(); + + for (int i = 0; i < ranges.Count; i++) + { + ulong lo = ranges.Start[i], hi = ranges.End[i]; + + while (lo <= hi) + { + int bits = 32; + while (bits > 0) + { + ulong size = 1UL << (32 - (bits - 1)); + if ((lo % size) != 0UL) break; + if (lo + size - 1UL > hi) break; + bits--; + } + + result.Add(bits == 32 ? FormatIp((uint)lo) : FormatCidr((uint)lo, bits)); + lo += 1UL << (32 - bits); + } + } + + return result.ToArray(); + } + static int WriteOctet(char[] buf, int pos, uint v) { if (v >= 100) { buf[pos++] = (char)('0' + v / 100); buf[pos++] = (char)('0' + (v / 10) % 10); } @@ -449,6 +928,37 @@ function Get-Url { finally { $resp.Close() } } +# --------------------------------------------------------------------------------------------------------- +# Refresh carve-out data from routing announcements. Runs here because it needs Get-Url and the compiled +# collapser. A failure leaves the existing file alone rather than truncating a working carve-out. +# --------------------------------------------------------------------------------------------------------- +if ($AddCarveout) { + $path = Join-Path $ConfigDir ("ip-allowlist-{0}.txt" -f $AddCarveout) + $prefixes = Get-CarveoutPrefixes -CarveoutAsn $Asn + + Write-AllowlistFile -Path $path -Lines (@(Get-CarveoutHeader -Name $AddCarveout -CarveoutAsn $Asn) + $prefixes) + Write-Host ("Created {0} carve-out from AS{1}: {2} prefixes -> {3}" -f $AddCarveout, $Asn, @($prefixes).Count, $path) + + if ($AllowPaths -notcontains $path) { $AllowPaths += $path } +} + +if ($RefreshCarveouts) { + foreach ($path in $AllowPaths) { + $carveoutAsn = Get-CarveoutAsn -Path $path + if ($carveoutAsn -le 0) { continue } # hand-written allowlist, not ours to rewrite + + try { + $name = [IO.Path]::GetFileNameWithoutExtension($path) -replace '^ip-allowlist-', '' + $prefixes = Get-CarveoutPrefixes -CarveoutAsn $carveoutAsn + Write-AllowlistFile -Path $path -Lines (@(Get-CarveoutHeader -Name $name -CarveoutAsn $carveoutAsn) + $prefixes) + Write-Host ("Refreshed {0} carve-out from AS{1}: {2} prefixes" -f $name, $carveoutAsn, @($prefixes).Count) + } + catch { + Write-Warning ("AS{0}: refresh failed ({1}) -- keeping the existing carve-out" -f $carveoutAsn, $_.Exception.Message) + } + } +} + # --------------------------------------------------------------------------------------------------------- # Collect every kept feed into ONE global set, timing each phase. # --------------------------------------------------------------------------------------------------------- @@ -483,18 +993,45 @@ foreach ($feed in $AllFeeds) { } # --------------------------------------------------------------------------------------------------------- -# Optional: subtract Tor / open-proxy / VPN IPs. +# Subtraction pass: the allowlist file, plus the anonymizer feeds when -ExcludeAnonymizers is set. Both are +# "never block these", so they share one set and one range-correct removal -- which is also the fix for the +# old anonymizer path, where CIDR entries were parsed and then never actually subtracted. +# +# Bogon filtering is deliberately NOT applied here: it exists to keep junk OUT of the blocklist, and running +# it over subtractive input would quietly discard exemptions instead (e.g. a shard exempting its own LAN). # --------------------------------------------------------------------------------------------------------- +$allowSingles = [System.Collections.Generic.HashSet[uint32]]::new() +$allowCidrs = [System.Collections.Generic.HashSet[string]]::new() +$noBogon = [uint32[]]::new(0) + +foreach ($p in $AllowPaths) { + try { + $allowText = Get-Content -LiteralPath $p -Raw -ErrorAction Stop + if (-not $allowText) { continue } + + $before = $allowSingles.Count + $allowCidrs.Count + [void][BlocklistExporter]::AddContent($allowText, $allowSingles, $allowCidrs, $noBogon, $noBogon) + Write-Host (" allowlist {0,-28} +{1} entr(ies)" -f (Split-Path -Leaf $p), (($allowSingles.Count + $allowCidrs.Count) - $before)) + } + catch { Write-Warning ("Could not read allowlist {0}: {1}" -f $p, $_.Exception.Message) } +} + if ($ExcludeAnonymizers) { - $anon = [System.Collections.Generic.HashSet[uint32]]::new() - $anonCidr = [System.Collections.Generic.HashSet[string]]::new() foreach ($url in $AnonFeeds) { - try { [void][BlocklistExporter]::AddContent((Get-Url -Url $url -Label 'anonymizers'), $anon, $anonCidr, $bogStart, $bogEnd) } + try { [void][BlocklistExporter]::AddContent((Get-Url -Url $url -Label 'anonymizers'), $allowSingles, $allowCidrs, $noBogon, $noBogon) } catch { Write-Warning ("anonymizer list {0}: {1}" -f $url, $_.Exception.Message) } } - $removed = 0 - foreach ($ip in @($anon)) { if ($singles.Remove($ip)) { $removed++ } } - Write-Host ("ExcludeAnonymizers: removed {0} Tor/proxy/VPN single IPs" -f $removed) +} + +$allowCount = $allowSingles.Count + $allowCidrs.Count +if ($allowCount -gt 0) { + $aSw = [System.Diagnostics.Stopwatch]::StartNew() + $ranges = [BlocklistExporter]::BuildRanges($allowSingles, $allowCidrs) + $res = [BlocklistExporter]::ApplyAllowlist($singles, $cidrs, $ranges) + $aSw.Stop() + + Write-Host ("Allowlist: {0} entr(ies) -> {1} ranges; removed {2} IPs, dropped {3} CIDRs, split {4} into {5} ({6:N1}s)" -f ` + $allowCount, $ranges.Count, $res.SinglesRemoved, $res.CidrsDropped, $res.CidrsSplit, $res.EntriesAdded, $aSw.Elapsed.TotalSeconds) } $total = $singles.Count + $cidrs.Count @@ -521,7 +1058,9 @@ if ($outDir -and -not (Test-Path -LiteralPath $outDir -PathType Container)) { # InvariantCulture: ':' is the culture-defined time separator in a custom format string, and the # header is a machine-read marker the shard compares verbatim. $generated = [DateTime]::UtcNow.ToString('yyyy-MM-ddTHH:mm:ssZ', [Globalization.CultureInfo]::InvariantCulture) -$header = "# modernuo-blocklist generated=$generated count=$total ipv4=$($singles.Count) cidr=$($cidrs.Count) feeds=$feedCount" +# The shard's header reader is token-based and ignores tokens it does not know, so `allow=` is additive -- +# it is here so an operator can tell from the file alone whether a carve-out was in effect when it was built. +$header = "# modernuo-blocklist generated=$generated count=$total ipv4=$($singles.Count) cidr=$($cidrs.Count) feeds=$feedCount allow=$allowCount" $tmp = $OutFile + '.tmp' $wSw = [System.Diagnostics.Stopwatch]::StartNew()