fix(core): Adds Hashed & Hierarchical Timer Wheel (#655)

- Removes TimerPriority
- Removes TimerThread
- Adds a [Hashed & Hierarchical Timer Wheel](http://www.cs.columbia.edu/~nahum/w6998/papers/ton97-timing-wheels.pdf)
This commit is contained in:
Kamron Batman 2021-07-11 22:42:06 -07:00 • committed by GitHub
parent b99d520019
commit fb915992dd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
138 changed files with 468 additions and 802 deletions

View file

@ -67,7 +67,7 @@ namespace Server
for (int i = 0; i < types.Length; i++)
{
var m = types[i].GetMethod(method, BindingFlags.Static | BindingFlags.Public);
if (m != null)
if (m?.GetParameters().Length == 0)
{
list.Add(m);
}

View file

@ -36,7 +36,6 @@ namespace Server
private static readonly ILogger logger = LogFactory.GetLogger(typeof(Core));
private static bool _crashed;
private static Thread _timerThread;
private static string _baseDirectory;
private static bool _profiling;
@ -127,8 +126,8 @@ namespace Server
// For Unix Stopwatch.Frequency is normalized to 1ns
// We don't anticipate needing this for Windows/OSX
private static long _maxTickCountBeforePrecisionLoss = long.MaxValue / 1000L;
private static long _ticksPerMillisecond = Stopwatch.Frequency / 1000L;
private const long _maxTickCountBeforePrecisionLoss = long.MaxValue / 1000L;
private static readonly long _ticksPerMillisecond = Stopwatch.Frequency / 1000L;
public static long TickCount
{
@ -145,6 +144,8 @@ namespace Server
// No precision loss
: 1000L * timestamp / Stopwatch.Frequency;
}
// Setting this to a value lower than the previous is bad. Timers will become delayed
// until time catches up.
set => _tickCount = value;
}
@ -358,8 +359,6 @@ namespace Server
{
EventSink.InvokeShutdown();
}
Timer.TimerThread.Set();
}
public static void Main(string[] args)
@ -420,12 +419,6 @@ namespace Server
logger.Information($"Running on {RuntimeInformation.FrameworkDescription}");
var ttObj = new Timer.TimerThread();
_timerThread = new Thread(ttObj.TimerMain)
{
Name = "Timer Thread"
};
var s = Arguments;
if (s.Length > 0)
@ -474,6 +467,8 @@ namespace Server
VerifySerialization();
Timer.Initialize(TickCount);
AssemblyHandler.Invoke("Configure");
TileMatrixLoader.LoadTileMatrix();
@ -483,8 +478,6 @@ namespace Server
AssemblyHandler.Invoke("Initialize");
_timerThread.Start();
TcpServer.Start();
EventSink.InvokeServerStarted();
RunEventLoop();
@ -504,7 +497,7 @@ namespace Server
var events = Mobile.ProcessDeltaQueue();
events += Item.ProcessDeltaQueue();
events += Timer.Slice();
events += Timer.Slice(_tickCount);
// Handle networking
events += TcpServer.Slice();

View file

@ -1157,16 +1157,6 @@ namespace Server
{
m_Player = value;
InvalidateProperties();
if (!m_Player && m_Dex <= 100 && m_CombatTimer != null)
{
m_CombatTimer.Priority = TimerPriority.FiftyMS;
}
else if (m_CombatTimer != null)
{
m_CombatTimer.Priority = TimerPriority.EveryTick;
}
CheckStatTimers();
}
}
@ -6546,15 +6536,6 @@ namespace Server
CheckStatTimers();
}
if (!m_Player && m_Dex <= 100 && m_CombatTimer != null)
{
m_CombatTimer.Priority = TimerPriority.FiftyMS;
}
else if (m_CombatTimer != null)
{
m_CombatTimer.Priority = TimerPriority.EveryTick;
}
UpdateRegion();
UpdateResistances();
@ -9375,7 +9356,6 @@ namespace Server
public ManaTimer(Mobile m)
: base(GetManaRegenRate(m), GetManaRegenRate(m))
{
Priority = TimerPriority.FiftyMS;
m_Owner = m;
}
@ -9397,7 +9377,6 @@ namespace Server
public HitsTimer(Mobile m)
: base(GetHitsRegenRate(m), GetHitsRegenRate(m))
{
Priority = TimerPriority.FiftyMS;
m_Owner = m;
}
@ -9419,7 +9398,6 @@ namespace Server
public StamTimer(Mobile m)
: base(GetStamRegenRate(m), GetStamRegenRate(m))
{
Priority = TimerPriority.FiftyMS;
m_Owner = m;
}
@ -9451,16 +9429,9 @@ namespace Server
{
private readonly Mobile m_Mobile;
public CombatTimer(Mobile m) : base(TimeSpan.FromSeconds(0.0), TimeSpan.FromSeconds(0.01))
{
public CombatTimer(Mobile m) : base(TimeSpan.FromSeconds(0.0), TimeSpan.FromSeconds(0.01)) =>
m_Mobile = m;
if (!m_Mobile.m_Player && m_Mobile.m_Dex <= 100)
{
Priority = TimerPriority.FiftyMS;
}
}
protected override void OnTick()
{
if (Core.TickCount - m_Mobile.NextCombatTime < 0)

View file

@ -39,9 +39,6 @@
</Target>
<ItemGroup>
<PackageReference Include="Microsoft.Toolkit.HighPerformance" Version="7.0.2" />
<PackageReference Include="Serilog" Version="2.10.0" />
<PackageReference Include="Serilog.Sinks.Async" Version="1.4.0" />
<PackageReference Include="Serilog.Sinks.Console" Version="3.1.1" />
<PackageReference Include="Zlib.Bindings" Version="1.5.0" />
</ItemGroup>
<ItemGroup>

View file

@ -262,23 +262,6 @@ namespace Server.Targeting
{
m_Target = target;
m_Mobile = m;
if (delay >= ThirtySeconds)
{
Priority = TimerPriority.FiveSeconds;
}
else if (delay >= TenSeconds)
{
Priority = TimerPriority.OneSecond;
}
else if (delay >= OneSecond)
{
Priority = TimerPriority.TwoFiftyMS;
}
else
{
Priority = TimerPriority.TwentyFiveMS;
}
}
protected override void OnTick()

View file

@ -29,6 +29,9 @@ namespace Server
public partial class Timer
{
private static string FormatDelegate(Delegate callback) =>
callback == null ? "null" : $"{callback.Method.DeclaringType?.FullName ?? ""}.{callback.Method.Name}";
public static Timer DelayCall(TimerCallback callback) => DelayCall(TimeSpan.Zero, TimeSpan.Zero, 1, callback);
public static Timer DelayCall(TimeSpan delay, TimerCallback callback) =>
@ -40,8 +43,6 @@ namespace Server
public static Timer DelayCall(TimeSpan delay, TimeSpan interval, int count, TimerCallback callback)
{
Timer t = new DelayCallTimer(delay, interval, count, callback);
t.Priority = ComputePriority(count == 1 ? delay : interval);
t.Start();
return t;
@ -62,9 +63,6 @@ namespace Server
)
{
Timer t = new DelayStateCallTimer<T>(delay, interval, count, callback, state);
t.Priority = ComputePriority(count == 1 ? delay : interval);
t.Start();
return t;
@ -87,9 +85,6 @@ namespace Server
)
{
Timer t = new DelayStateCallTimer<T1, T2>(delay, interval, count, callback, t1, t2);
t.Priority = ComputePriority(count == 1 ? delay : interval);
t.Start();
return t;
@ -114,9 +109,6 @@ namespace Server
)
{
Timer t = new DelayStateCallTimer<T1, T2, T3>(delay, interval, count, callback, t1, t2, t3);
t.Priority = ComputePriority(count == 1 ? delay : interval);
t.Start();
return t;
@ -144,9 +136,6 @@ namespace Server
)
{
Timer t = new DelayStateCallTimer<T1, T2, T3, T4>(delay, interval, count, callback, t1, t2, t3, t4);
t.Priority = ComputePriority(count == 1 ? delay : interval);
t.Start();
return t;
@ -161,13 +150,10 @@ namespace Server
)
{
Callback = callback;
RegCreation();
}
public TimerCallback Callback { get; }
public override bool DefRegCreation => false;
protected override void OnTick()
{
Callback?.Invoke();
@ -185,14 +171,10 @@ namespace Server
{
Callback = callback;
m_State = state;
RegCreation();
}
public TimerStateCallback<T> Callback { get; }
public override bool DefRegCreation => false;
protected override void OnTick()
{
Callback?.Invoke(m_State);
@ -214,14 +196,10 @@ namespace Server
Callback = callback;
m_T1 = t1;
m_T2 = t2;
RegCreation();
}
public TimerStateCallback<T1, T2> Callback { get; }
public override bool DefRegCreation => false;
protected override void OnTick()
{
Callback?.Invoke(m_T1, m_T2);
@ -245,14 +223,10 @@ namespace Server
m_T1 = t1;
m_T2 = t2;
m_T3 = t3;
RegCreation();
}
public TimerStateCallback<T1, T2, T3> Callback { get; }
public override bool DefRegCreation => false;
protected override void OnTick()
{
Callback?.Invoke(m_T1, m_T2, m_T3);
@ -278,14 +252,10 @@ namespace Server
m_T2 = t2;
m_T3 = t3;
m_T4 = t4;
RegCreation();
}
public TimerStateCallback<T1, T2, T3, T4> Callback { get; }
public override bool DefRegCreation => false;
protected override void OnTick()
{
Callback?.Invoke(m_T1, m_T2, m_T3, m_T4);

View file

@ -0,0 +1,263 @@
/*************************************************************************
* ModernUO *
* Copyright 2019-2021 - ModernUO Development Team *
* Email: hi@modernuo.com *
* File: Timer.TimerWheel.cs *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation, either version 3 of the License, or *
* (at your option) any later version. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
*************************************************************************/
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
namespace Server
{
public partial class Timer
{
private const int _ringSizePowerOf2 = 12;
private const int _ringSize = 1 << _ringSizePowerOf2; // 4096
private const int _ringLayers = 3;
private const int _tickRatePowerOf2 = 3;
private const int _tickRate = 1 << _tickRatePowerOf2; // 8ms
private static long _lastTickTurned = -1;
private static readonly Timer[][] _rings = new Timer[_ringLayers][];
private static readonly int[] _ringIndexes = new int[_ringLayers];
public static void Initialize(long tickCount)
{
_lastTickTurned = tickCount;
for (int i = 0; i < _rings.Length; i++)
{
_rings[i] = new Timer[_ringSize];
_ringIndexes[i] = 0;
}
}
public static int Slice(long tickCount)
{
var deltaSinceTurn = tickCount - _lastTickTurned;
var events = 0;
while (deltaSinceTurn >= _tickRate)
{
deltaSinceTurn -= _tickRate;
_lastTickTurned += _tickRate;
events += Turn() ? 1 : 0;
}
return events;
}
private static bool Turn()
{
bool events = false;
for (var i = 0; i < _ringLayers; i++)
{
// Increment the ring index, then get the timer.
var ringIndex = ++_ringIndexes[i];
bool turnNextWheel = ringIndex >= _ringSize;
if (turnNextWheel)
{
ringIndex = _ringIndexes[i] = 0;
}
var timer = _rings[i][ringIndex];
if (timer != null)
{
events = true;
if (i > 0 && timer._remaining > 0)
{
Promote(timer);
}
else
{
Execute(timer);
}
// Clear the slot
_rings[i][ringIndex] = null;
}
if (!turnNextWheel)
{
break;
}
}
return events;
}
private static void Execute(Timer timer)
{
do
{
var next = timer._nextTimer;
var prof = timer.GetProfile();
prof?.Start();
timer.OnTick();
prof?.Finish();
if (timer.Running)
{
RemoveTimer(timer);
if (timer.Count == 0 || ++timer.Index < timer.Count)
{
timer.Delay = timer.Interval;
timer.Next = Core.Now + timer.Interval;
AddTimer(timer, (long)timer.Delay.TotalMilliseconds);
}
}
timer = next;
} while (timer != null);
}
private static void Promote(Timer timer)
{
do
{
var next = timer._nextTimer;
RemoveTimer(timer);
AddTimer(timer, timer._remaining);
timer = next;
} while (timer != null);
}
private static void AddTimer(Timer timer, long delay)
{
delay = Math.Max(0, delay);
var resolutionPowerOf2 = _tickRatePowerOf2;
for (var i = 0; i < _ringLayers; i++)
{
var resolution = 1 << resolutionPowerOf2;
var nextResolutionPowerOf2 = resolutionPowerOf2 + _ringSizePowerOf2;
long max = 1 << nextResolutionPowerOf2;
if (delay < max)
{
var remaining = delay & (resolution - 1);
var slot = (delay >> resolutionPowerOf2) + _ringIndexes[i] + (remaining > 0 ? 1 : 0);
// Round up if we have a delay of 0
if (delay == 0)
{
slot++;
remaining = 0;
}
if (slot >= _ringSize)
{
slot -= _ringSize;
}
timer._nextTimer = _rings[i][slot];
if (timer._nextTimer != null)
{
timer._nextTimer._prevTimer = timer;
}
timer._remaining = remaining;
timer._ring = i;
timer._slot = (int)slot;
_rings[i][slot] = timer;
break;
}
// The remaining amount until we turn this ring
delay -= resolution * (_ringSize - _ringIndexes[i]);
resolutionPowerOf2 = nextResolutionPowerOf2;
}
}
private static void RemoveTimer(Timer timer)
{
if (timer._prevTimer != null)
{
timer._prevTimer._nextTimer = timer._nextTimer;
}
else
{
_rings[timer._ring][timer._slot] = timer._nextTimer;
}
if (timer._nextTimer != null)
{
timer._nextTimer._prevTimer = timer._prevTimer;
}
timer._nextTimer = null;
timer._prevTimer = null;
}
public static void DumpInfo(TextWriter tw)
{
var now = DateTime.UtcNow;
tw.WriteLine("Date: {0}", now);
tw.WriteLine();
var total = 0.0;
var hash = new Dictionary<string, int>();
for (var i = 0; i < _ringLayers; i++)
{
for (var j = 0; j < _ringSize; j++)
{
var t = _rings[i][j];
var name = t.ToString();
hash.TryGetValue(name, out var count);
hash[name] = count + 1;
total++;
}
}
foreach (var (name, count) in hash.OrderByDescending(o => o.Value))
{
tw.WriteLine($"Type: {name}; Count: {count}; Percent: {count / total}%");
}
tw.WriteLine();
tw.WriteLine();
}
public static void ClearAllTimers(long tickCount)
{
_lastTickTurned = tickCount;
foreach (var t in _rings)
{
for (var i = 0; i < _ringSize; i++)
{
var node = t[i];
Timer next;
do
{
next = node?._nextTimer;
node?.Stop();
} while (next != null);
}
}
}
}
}

View file

@ -14,39 +14,19 @@
*************************************************************************/
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using Server.Diagnostics;
namespace Server
{
public enum TimerPriority
{
EveryTick,
TenMS,
TwentyFiveMS,
FiftyMS,
TwoFiftyMS,
OneSecond,
FiveSeconds,
OneMinute
}
public partial class Timer
{
private static readonly Queue<Timer> m_Queue = new();
// We need to know what ring/slot we are in so we can be removed if we are "head" of the link list.
private int _ring;
private int _slot;
private static int m_QueueCountAtSlice;
private long m_Delay;
private long m_Interval;
private List<Timer> m_List;
private long m_Next;
private TimerPriority m_Priority;
private bool m_PrioritySet;
private bool m_Queued;
private bool m_Running;
private long _remaining;
private Timer _nextTimer;
private Timer _prevTimer;
public Timer(TimeSpan delay) : this(delay, TimeSpan.Zero, 1)
{
@ -54,130 +34,13 @@ namespace Server
public Timer(TimeSpan delay, TimeSpan interval, int count = 0)
{
m_Delay = (long)delay.TotalMilliseconds;
m_Interval = (long)interval.TotalMilliseconds;
Delay = delay;
Interval = interval;
Count = count;
_nextTimer = null;
_prevTimer = null;
Next = Core.Now + Delay;
if (!m_PrioritySet)
{
m_Priority = ComputePriority(count == 1 ? delay : interval);
m_PrioritySet = true;
}
if (DefRegCreation)
{
RegCreation();
}
}
public TimerPriority Priority
{
get => m_Priority;
set
{
if (!m_PrioritySet)
{
m_PrioritySet = true;
}
if (m_Priority != value)
{
m_Priority = value;
if (m_Running)
{
TimerThread.PriorityChange(this, (int)m_Priority);
}
}
}
}
public DateTime Next => DateTime.UtcNow + TimeSpan.FromMilliseconds(m_Next - Core.TickCount);
public TimeSpan Delay
{
get => TimeSpan.FromMilliseconds(m_Delay);
set => m_Delay = (long)value.TotalMilliseconds;
}
public TimeSpan Interval
{
get => TimeSpan.FromMilliseconds(m_Interval);
set => m_Interval = (long)value.TotalMilliseconds;
}
public int Index { get; private set; }
public int Count { get; }
public bool Running
{
get => m_Running;
set
{
if (value)
{
Start();
}
else
{
Stop();
}
}
}
public static int BreakCount { get; set; } = 20000;
public virtual bool DefRegCreation => true;
private static string FormatDelegate(Delegate callback) =>
callback == null ? "null" : $"{callback.Method.DeclaringType?.FullName ?? ""}.{callback.Method.Name}";
public static void DumpInfo(TextWriter tw)
{
TimerThread.DumpInfo(tw);
}
public TimerProfile GetProfile()
{
if (!Core.Profiling)
{
return null;
}
var name = ToString();
return TimerProfile.Acquire(name);
}
public static int Slice()
{
var index = 0;
lock (m_Queue)
{
m_QueueCountAtSlice = m_Queue.Count;
while (index < BreakCount && m_Queue.Count != 0)
{
var t = m_Queue.Dequeue();
var prof = t.GetProfile();
prof?.Start();
t.OnTick();
t.m_Queued = false;
index++;
prof?.Finish();
}
}
return index;
}
public void RegCreation()
{
var prof = GetProfile();
if (prof != null)
@ -186,321 +49,59 @@ namespace Server
}
}
public override string ToString() => GetType().FullName ?? "";
public DateTime Next { get; private set; }
public TimeSpan Delay { get; set; }
public TimeSpan Interval { get; set; }
public int Index { get; private set; }
public int Count { get; }
public bool Running { get; private set; }
public static TimerPriority ComputePriority(TimeSpan ts)
public TimerProfile GetProfile() => !Core.Profiling ? null : TimerProfile.Acquire(ToString() ?? "null");
public override string ToString() => GetType().FullName;
public Timer Start()
{
if (ts >= TimeSpan.FromMinutes(1.0))
if (Running)
{
return TimerPriority.FiveSeconds;
return this;
}
if (ts >= TimeSpan.FromSeconds(10.0))
Running = true;
AddTimer(this, (long)Delay.TotalMilliseconds);
var prof = GetProfile();
if (prof != null)
{
return TimerPriority.OneSecond;
prof.Started++;
}
if (ts >= TimeSpan.FromSeconds(5.0))
{
return TimerPriority.TwoFiftyMS;
}
if (ts >= TimeSpan.FromSeconds(2.5))
{
return TimerPriority.FiftyMS;
}
if (ts >= TimeSpan.FromSeconds(1.0))
{
return TimerPriority.TwentyFiveMS;
}
if (ts >= TimeSpan.FromSeconds(0.5))
{
return TimerPriority.TenMS;
}
return TimerPriority.EveryTick;
return this;
}
public void Start()
public Timer Stop()
{
if (!m_Running)
if (!Running)
{
m_Running = true;
TimerThread.AddTimer(this);
var prof = GetProfile();
if (prof != null)
{
prof.Started++;
}
return this;
}
}
public void Stop()
{
if (m_Running)
Running = false;
RemoveTimer(this);
var prof = GetProfile();
if (prof != null)
{
m_Running = false;
TimerThread.RemoveTimer(this);
var prof = GetProfile();
if (prof != null)
{
prof.Stopped++;
}
prof.Stopped++;
}
return this;
}
protected virtual void OnTick()
{
}
public class TimerThread
{
private static readonly Dictionary<Timer, TimerChangeEntry>
m_Changed = new();
private static readonly long[] m_NextPriorities = new long[8];
private static readonly long[] m_PriorityDelays =
{
0,
10,
25,
50,
250,
1000,
5000,
60000
};
private static readonly List<Timer>[] m_Timers =
{
new(),
new(),
new(),
new(),
new(),
new(),
new(),
new()
};
private static readonly AutoResetEvent m_Signal = new(false);
public static void DumpInfo(TextWriter tw)
{
for (var i = 0; i < 8; ++i)
{
tw.WriteLine("Priority: {0}", (TimerPriority)i);
tw.WriteLine();
var hash = new Dictionary<string, List<Timer>>();
for (var j = 0; j < m_Timers[i].Count; ++j)
{
var t = m_Timers[i][j];
var key = t.ToString();
if (!hash.TryGetValue(key, out var list))
{
hash[key] = list = new List<Timer>();
}
list.Add(t);
}
foreach (var kv in hash)
{
var key = kv.Key;
var list = kv.Value;
tw.WriteLine(
"Type: {0}; Count: {1}; Percent: {2}%",
key,
list.Count,
(int)(100 * (list.Count / (double)m_Timers[i].Count))
);
}
tw.WriteLine();
tw.WriteLine();
}
}
public static void Change(Timer t, int newIndex, bool isAdd)
{
lock (m_Changed)
{
m_Changed[t] = TimerChangeEntry.GetInstance(t, newIndex, isAdd);
}
m_Signal.Set();
}
public static void AddTimer(Timer t)
{
Change(t, (int)t.Priority, true);
}
public static void PriorityChange(Timer t, int newPrio)
{
Change(t, newPrio, false);
}
public static void RemoveTimer(Timer t)
{
Change(t, -1, false);
}
private static void ProcessChanged()
{
lock (m_Changed)
{
var curTicks = Core.TickCount;
foreach (var tce in m_Changed.Values)
{
var timer = tce.m_Timer;
var newIndex = tce.m_NewIndex;
timer.m_List?.Remove(timer);
if (tce.m_IsAdd)
{
timer.m_Next = curTicks + timer.m_Delay;
timer.Index = 0;
}
if (newIndex >= 0)
{
timer.m_List = m_Timers[newIndex];
timer.m_List.Add(timer);
}
else
{
timer.m_List = null;
}
tce.Free();
}
m_Changed.Clear();
}
}
public static void Set()
{
m_Signal.Set();
}
public void TimerMain()
{
while (!Core.Closing)
{
if (World.Loading || World.Saving)
{
m_Signal.WaitOne(1, false);
continue;
}
ProcessChanged();
for (var i = 0; i < m_Timers.Length; i++)
{
var now = Core.TickCount;
if (now < m_NextPriorities[i])
{
break;
}
m_NextPriorities[i] = now + m_PriorityDelays[i];
for (var j = 0; j < m_Timers[i].Count; j++)
{
var t = m_Timers[i][j];
if (!t.m_Queued && now > t.m_Next)
{
t.m_Queued = true;
lock (m_Queue)
{
m_Queue.Enqueue(t);
}
if (t.Count != 0 && ++t.Index >= t.Count)
{
t.Stop();
}
else
{
t.m_Next = now + t.m_Interval;
}
}
}
}
m_Signal.WaitOne(1, false);
}
}
private class TimerChangeEntry
{
private static readonly Queue<TimerChangeEntry> m_InstancePool = new();
public bool m_IsAdd;
public int m_NewIndex;
public Timer m_Timer;
private TimerChangeEntry(Timer t, int newIndex, bool isAdd)
{
m_Timer = t;
m_NewIndex = newIndex;
m_IsAdd = isAdd;
}
public void Free()
{
lock (m_InstancePool)
{
if (m_InstancePool.Count < 200) // Arbitrary
{
m_InstancePool.Enqueue(this);
}
}
}
public static TimerChangeEntry GetInstance(Timer t, int newIndex, bool isAdd)
{
TimerChangeEntry e = null;
lock (m_InstancePool)
{
if (m_InstancePool.Count > 0)
{
e = m_InstancePool.Dequeue();
}
}
if (e != null)
{
e.m_Timer = t;
e.m_NewIndex = newIndex;
e.m_IsAdd = isAdd;
}
else
{
e = new TimerChangeEntry(t, newIndex, isAdd);
}
return e;
}
}
}
}
}